From 7abfc17a6d1334a885e16d07aa3f6d1c875279c8 Mon Sep 17 00:00:00 2001 From: Ketan Verma <9292653+ketanv3@users.noreply.github.com> Date: Fri, 8 Sep 2023 17:16:41 +0530 Subject: [PATCH 001/388] Improve performance of rounding dates in date_histogram aggregation (#9727) * Improve performance of rounding dates in date_histogram aggregation Signed-off-by: Ketan Verma * Minor refactoring changes Signed-off-by: Ketan Verma --------- Signed-off-by: Ketan Verma --- CHANGELOG.md | 1 + .../common/ArrayRoundingBenchmark.java | 147 ++++++++++++++++++ .../java/org/opensearch/common/Rounding.java | 100 ++++++++++-- .../org/opensearch/common/RoundingTests.java | 22 +++ 4 files changed, 261 insertions(+), 9 deletions(-) create mode 100644 benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 076eebb65f19c..ce67273f2a6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -182,6 +182,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Redefine telemetry context restoration and propagation ([#9617](https://github.com/opensearch-project/OpenSearch/pull/9617)) - Use non-concurrent path for sort request on timeseries index and field([#9562](https://github.com/opensearch-project/OpenSearch/pull/9562)) - Added sampler based on `Blanket Probabilistic Sampling rate` and `Override for on demand` ([#9621](https://github.com/opensearch-project/OpenSearch/issues/9621)) +- Improve performance of rounding dates in date_histogram aggregation ([#9727](https://github.com/opensearch-project/OpenSearch/pull/9727)) - [Remote Store] Add support for Remote Translog Store stats in `_remotestore/stats/` API ([#9263](https://github.com/opensearch-project/OpenSearch/pull/9263)) - Add support for query profiler with concurrent aggregation ([#9248](https://github.com/opensearch-project/OpenSearch/pull/9248)) - Cleanup Unreferenced file on segment merge failure ([#9503](https://github.com/opensearch-project/OpenSearch/pull/9503)) diff --git a/benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java b/benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java new file mode 100644 index 0000000000000..64c0a9e1d7aa6 --- /dev/null +++ b/benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.Random; +import java.util.function.Supplier; + +@Fork(value = 3) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 1, time = 1) +@BenchmarkMode(Mode.Throughput) +public class ArrayRoundingBenchmark { + + @Benchmark + public void round(Blackhole bh, Options opts) { + Rounding.Prepared rounding = opts.supplier.get(); + for (long key : opts.queries) { + bh.consume(rounding.round(key)); + } + } + + @State(Scope.Benchmark) + public static class Options { + @Param({ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "12", + "14", + "16", + "18", + "20", + "22", + "24", + "26", + "29", + "32", + "37", + "41", + "45", + "49", + "54", + "60", + "64", + "74", + "83", + "90", + "98", + "108", + "118", + "128", + "144", + "159", + "171", + "187", + "204", + "229", + "256" }) + public Integer size; + + @Param({ "binary", "linear" }) + public String type; + + @Param({ "uniform", "skewed_edge", "skewed_center" }) + public String distribution; + + public long[] queries; + public Supplier supplier; + + @Setup + public void setup() { + Random random = new Random(size); + long[] values = new long[size]; + for (int i = 1; i < values.length; i++) { + values[i] = values[i - 1] + 100; + } + + long range = values[values.length - 1] - values[0] + 100; + long mean, stddev; + queries = new long[1000000]; + + switch (distribution) { + case "uniform": // all values equally likely. + for (int i = 0; i < queries.length; i++) { + queries[i] = values[0] + (nextPositiveLong(random) % range); + } + break; + case "skewed_edge": // distribution centered at p90 with ± 5% stddev. + mean = values[0] + (long) (range * 0.9); + stddev = (long) (range * 0.05); + for (int i = 0; i < queries.length; i++) { + queries[i] = Math.max(values[0], mean + (long) (random.nextGaussian() * stddev)); + } + break; + case "skewed_center": // distribution centered at p50 with ± 5% stddev. + mean = values[0] + (long) (range * 0.5); + stddev = (long) (range * 0.05); + for (int i = 0; i < queries.length; i++) { + queries[i] = Math.max(values[0], mean + (long) (random.nextGaussian() * stddev)); + } + break; + default: + throw new IllegalArgumentException("invalid distribution: " + distribution); + } + + switch (type) { + case "binary": + supplier = () -> new Rounding.BinarySearchArrayRounding(values, size, null); + break; + case "linear": + supplier = () -> new Rounding.BidirectionalLinearSearchArrayRounding(values, size, null); + break; + default: + throw new IllegalArgumentException("invalid type: " + type); + } + } + + private static long nextPositiveLong(Random random) { + return random.nextLong() & Long.MAX_VALUE; + } + } +} diff --git a/server/src/main/java/org/opensearch/common/Rounding.java b/server/src/main/java/org/opensearch/common/Rounding.java index 65ffdafc423fd..667eb4529fe38 100644 --- a/server/src/main/java/org/opensearch/common/Rounding.java +++ b/server/src/main/java/org/opensearch/common/Rounding.java @@ -37,6 +37,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.common.LocalTimeOffset.Gap; import org.opensearch.common.LocalTimeOffset.Overlap; +import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.time.DateUtils; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; @@ -412,6 +413,21 @@ public Rounding build() { } private abstract class PreparedRounding implements Prepared { + /** + * The maximum limit up to which array-based prepared rounding is used. + * 128 is a power of two that isn't huge. We might be able to do + * better if the limit was based on the actual type of prepared + * rounding but this'll do for now. + */ + private static final int DEFAULT_ARRAY_ROUNDING_MAX_THRESHOLD = 128; + + /** + * The maximum limit up to which linear search is used, otherwise binary search is used. + * This is because linear search is much faster on small arrays. + * Benchmark results: PR #9727 + */ + private static final int LINEAR_SEARCH_ARRAY_ROUNDING_MAX_THRESHOLD = 64; + /** * Attempt to build a {@link Prepared} implementation that relies on pre-calcuated * "round down" points. If there would be more than {@code max} points then return @@ -435,7 +451,9 @@ protected Prepared maybeUseArray(long minUtcMillis, long maxUtcMillis, int max) values = ArrayUtil.grow(values, i + 1); values[i++] = rounded; } - return new ArrayRounding(values, i, this); + return i <= LINEAR_SEARCH_ARRAY_ROUNDING_MAX_THRESHOLD + ? new BidirectionalLinearSearchArrayRounding(values, i, this) + : new BinarySearchArrayRounding(values, i, this); } } @@ -521,12 +539,11 @@ private LocalDateTime truncateLocalDateTime(LocalDateTime localDateTime) { @Override public Prepared prepare(long minUtcMillis, long maxUtcMillis) { - /* - * 128 is a power of two that isn't huge. We might be able to do - * better if the limit was based on the actual type of prepared - * rounding but this'll do for now. - */ - return prepareOffsetOrJavaTimeRounding(minUtcMillis, maxUtcMillis).maybeUseArray(minUtcMillis, maxUtcMillis, 128); + return prepareOffsetOrJavaTimeRounding(minUtcMillis, maxUtcMillis).maybeUseArray( + minUtcMillis, + maxUtcMillis, + PreparedRounding.DEFAULT_ARRAY_ROUNDING_MAX_THRESHOLD + ); } private TimeUnitPreparedRounding prepareOffsetOrJavaTimeRounding(long minUtcMillis, long maxUtcMillis) { @@ -1330,14 +1347,19 @@ public static Rounding read(StreamInput in) throws IOException { /** * Implementation of {@link Prepared} using pre-calculated "round down" points. * + *

+ * It uses binary search to find the greatest round-down point less than or equal to the given timestamp. + * * @opensearch.internal */ - private static class ArrayRounding implements Prepared { + @InternalApi + static class BinarySearchArrayRounding implements Prepared { private final long[] values; private final int max; private final Prepared delegate; - private ArrayRounding(long[] values, int max, Prepared delegate) { + BinarySearchArrayRounding(long[] values, int max, Prepared delegate) { + assert max > 0 : "at least one round-down point must be present"; this.values = values; this.max = max; this.delegate = delegate; @@ -1365,4 +1387,64 @@ public double roundingSize(long utcMillis, DateTimeUnit timeUnit) { return delegate.roundingSize(utcMillis, timeUnit); } } + + /** + * Implementation of {@link Prepared} using pre-calculated "round down" points. + * + *

+ * It uses linear search to find the greatest round-down point less than or equal to the given timestamp. + * For small inputs (≤ 64 elements), this can be much faster than binary search as it avoids the penalty of + * branch mispredictions and pipeline stalls, and accesses memory sequentially. + * + *

+ * It uses "meet in the middle" linear search to avoid the worst case scenario when the desired element is present + * at either side of the array. This is helpful for time-series data where velocity increases over time, so more + * documents are likely to find a greater timestamp which is likely to be present on the right end of the array. + * + * @opensearch.internal + */ + @InternalApi + static class BidirectionalLinearSearchArrayRounding implements Prepared { + private final long[] ascending; + private final long[] descending; + private final Prepared delegate; + + BidirectionalLinearSearchArrayRounding(long[] values, int max, Prepared delegate) { + assert max > 0 : "at least one round-down point must be present"; + this.delegate = delegate; + int len = (max + 1) >>> 1; // rounded-up to handle odd number of values + ascending = new long[len]; + descending = new long[len]; + + for (int i = 0; i < len; i++) { + ascending[i] = values[i]; + descending[i] = values[max - i - 1]; + } + } + + @Override + public long round(long utcMillis) { + int i = 0; + for (; i < ascending.length; i++) { + if (descending[i] <= utcMillis) { + return descending[i]; + } + if (ascending[i] > utcMillis) { + assert i > 0 : "utcMillis must be after " + ascending[0]; + return ascending[i - 1]; + } + } + return ascending[i - 1]; + } + + @Override + public long nextRoundingValue(long utcMillis) { + return delegate.nextRoundingValue(utcMillis); + } + + @Override + public double roundingSize(long utcMillis, DateTimeUnit timeUnit) { + return delegate.roundingSize(utcMillis, timeUnit); + } + } } diff --git a/server/src/test/java/org/opensearch/common/RoundingTests.java b/server/src/test/java/org/opensearch/common/RoundingTests.java index e0c44e3516e7b..0ebfe02dc7641 100644 --- a/server/src/test/java/org/opensearch/common/RoundingTests.java +++ b/server/src/test/java/org/opensearch/common/RoundingTests.java @@ -1143,6 +1143,28 @@ public void testNonMillisecondsBasedUnitCalendarRoundingSize() { assertThat(prepared.roundingSize(thirdQuarter, Rounding.DateTimeUnit.HOUR_OF_DAY), closeTo(2208.0, 0.000001)); } + public void testArrayRoundingImplementations() { + int length = randomIntBetween(1, 256); + long[] values = new long[length]; + for (int i = 1; i < values.length; i++) { + values[i] = values[i - 1] + (randomNonNegativeLong() % 100); + } + + Rounding.Prepared binarySearchImpl = new Rounding.BinarySearchArrayRounding(values, length, null); + Rounding.Prepared linearSearchImpl = new Rounding.BidirectionalLinearSearchArrayRounding(values, length, null); + + for (int i = 0; i < 100000; i++) { + long key = values[0] + (randomNonNegativeLong() % (100 + values[length - 1] - values[0])); + assertEquals(binarySearchImpl.round(key), linearSearchImpl.round(key)); + } + + AssertionError exception = expectThrows(AssertionError.class, () -> { binarySearchImpl.round(values[0] - 1); }); + assertEquals("utcMillis must be after " + values[0], exception.getMessage()); + + exception = expectThrows(AssertionError.class, () -> { linearSearchImpl.round(values[0] - 1); }); + assertEquals("utcMillis must be after " + values[0], exception.getMessage()); + } + private void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes, ZoneId tz) { assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz); long millisPerMinute = 60_000; From 8e5e54b3a0d4e9c49f6ff062af361cf88ec76322 Mon Sep 17 00:00:00 2001 From: Neetika Singhal Date: Fri, 8 Sep 2023 06:16:27 -0700 Subject: [PATCH 002/388] Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation (#9916) Signed-off-by: Neetika Singhal Signed-off-by: Andriy Redko Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + .../search/aggregations/MissingValueIT.java | 28 ++++++++++++++++++- .../test/OpenSearchIntegTestCase.java | 18 ++++++++++-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce67273f2a6f8..d6b4660d85d49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -188,6 +188,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Cleanup Unreferenced file on segment merge failure ([#9503](https://github.com/opensearch-project/OpenSearch/pull/9503)) - Move ZStd to a plugin ([#9658](https://github.com/opensearch-project/OpenSearch/pull/9658)) - [Remote Store] Add support for Remote Translog Store upload stats in `_nodes/stats/` API ([#8908](https://github.com/opensearch-project/OpenSearch/pull/8908)) +- Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916)) - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java index 26bfe59618275..e6325987d330f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.terms.Terms; @@ -43,7 +47,12 @@ import org.opensearch.search.aggregations.metrics.Percentiles; import org.opensearch.search.aggregations.metrics.Stats; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.cardinality; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; @@ -56,7 +65,24 @@ import static org.hamcrest.Matchers.closeTo; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class MissingValueIT extends OpenSearchIntegTestCase { +public class MissingValueIT extends ParameterizedOpenSearchIntegTestCase { + + public MissingValueIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected int maximumNumberOfShards() { diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index f06a6e26defeb..65e5a18c89949 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2316,8 +2316,12 @@ private static void initializeSuiteScope() throws Exception { */ assert INSTANCE == null; if (isSuiteScopedTest(targetClass)) { - // note we need to do this this way to make sure this is reproducible - INSTANCE = (OpenSearchIntegTestCase) targetClass.getConstructor().newInstance(); + // note we need to do this way to make sure this is reproducible + if (isSuiteScopedTestParameterized(targetClass)) { + INSTANCE = (OpenSearchIntegTestCase) targetClass.getConstructor(Settings.class).newInstance(Settings.EMPTY); + } else { + INSTANCE = (OpenSearchIntegTestCase) targetClass.getConstructor().newInstance(); + } boolean success = false; try { INSTANCE.printTestMessage("setup"); @@ -2412,6 +2416,16 @@ private static boolean isSuiteScopedTest(Class clazz) { return clazz.getAnnotation(SuiteScopeTestCase.class) != null; } + /* + * For tests defined with, SuiteScopeTestCase return true if the + * class has a constructor that takes a single Settings parameter + * */ + private static boolean isSuiteScopedTestParameterized(Class clazz) { + return Arrays.stream(clazz.getConstructors()) + .filter(x -> x.getParameterTypes().length == 1) + .anyMatch(x -> x.getParameterTypes()[0].equals(Settings.class)); + } + /** * If a test is annotated with {@link SuiteScopeTestCase} * the checks and modifications that are applied to the used test cluster are only done after all tests From caf4c8062484331a225fa15811a674219fd6a6fa Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Fri, 8 Sep 2023 10:26:49 -0700 Subject: [PATCH 003/388] [Segment Replication] Support realtime TermVector requests with Segment Replication (#9585) * support realtime TermVector and MultiTermVector requests with segment replication. Signed-off-by: Rishikesh1159 * Fix TermVector requests with segrep. Signed-off-by: Rishikesh1159 * Refacotring. Signed-off-by: Rishikesh1159 * Address comments on PR. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 Signed-off-by: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> --- .../replication/SegmentReplicationIT.java | 155 ++++++++++++++++++ .../TransportTermVectorsAction.java | 14 +- 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 5855ed7470559..33bc5a8f3afe6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -15,6 +15,7 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentInfos; @@ -38,6 +39,8 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.termvectors.TermVectorsRequestBuilder; +import org.opensearch.action.termvectors.TermVectorsResponse; import org.opensearch.action.update.UpdateResponse; import org.opensearch.client.Requests; import org.opensearch.cluster.ClusterState; @@ -57,6 +60,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexModule; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationPressureService; @@ -81,6 +85,7 @@ import org.opensearch.transport.TransportService; import org.junit.Before; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1622,4 +1627,154 @@ public void testRealtimeMultiGetRequestsUnsuccessful() { assertTrue(mgetResponse.getResponses()[1].isFailed()); } + + /** + * Tests whether segment replication supports realtime termvector requests and reads and parses source from the translog to serve strong reads. + */ + public void testRealtimeTermVectorRequestsSuccessful() throws IOException { + final String primary = internalCluster().startDataOnlyNode(); + XContentBuilder mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", "text") + .field("term_vector", "with_positions_offsets_payloads") + .field("analyzer", "tv_test") + .endObject() + .endObject() + .endObject(); + // refresh interval disabled to ensure refresh rate of index (when data is ready for search) doesn't affect realtime termvectors + assertAcked( + prepareCreate(INDEX_NAME).setMapping(mapping) + .addAlias(new Alias("alias")) + .setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.analysis.analyzer.tv_test.tokenizer", "standard") + .put("index.refresh_interval", -1) + .putList("index.analysis.analyzer.tv_test.filter", "lowercase") + ) + ); + final String replica = internalCluster().startDataOnlyNode(); + ensureGreen(INDEX_NAME); + final String id = routingKeyForShard(INDEX_NAME, 0); + + TermVectorsResponse response = client(replica).prepareTermVectors(indexOrAlias(), "1").get(); + assertFalse(response.isExists()); + + // index doc 1 + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(1)) + .setSource(jsonBuilder().startObject().field("field", "the quick brown fox jumps over the lazy dog").endObject()) + .execute() + .actionGet(); + + // non realtime termvectors 1 + response = client().prepareTermVectors(indexOrAlias(), Integer.toString(1)).setRealtime(false).get(); + assertFalse(response.isExists()); + + // realtime termvectors 1 + TermVectorsRequestBuilder resp = client().prepareTermVectors(indexOrAlias(), Integer.toString(1)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setRealtime(true) + .setSelectedFields(); + response = resp.execute().actionGet(); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + assertThat("doc id: " + 1 + " doesn't exists but should", response.isExists(), equalTo(true)); + Fields fields = response.getFields(); + assertThat(fields.size(), equalTo(1)); + + // index doc 2 with routing + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(2)) + .setRouting(id) + .setSource(jsonBuilder().startObject().field("field", "the quick brown fox jumps over the lazy dog").endObject()) + .execute() + .actionGet(); + + // realtime termvectors 2 with routing + resp = client().prepareTermVectors(indexOrAlias(), Integer.toString(2)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setRouting(id) + .setSelectedFields(); + response = resp.execute().actionGet(); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + assertThat("doc id: " + 1 + " doesn't exists but should", response.isExists(), equalTo(true)); + fields = response.getFields(); + assertThat(fields.size(), equalTo(1)); + + } + + public void testRealtimeTermVectorRequestsUnSuccessful() throws IOException { + final String primary = internalCluster().startDataOnlyNode(); + XContentBuilder mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", "text") + .field("term_vector", "with_positions_offsets_payloads") + .field("analyzer", "tv_test") + .endObject() + .endObject() + .endObject(); + // refresh interval disabled to ensure refresh rate of index (when data is ready for search) doesn't affect realtime termvectors + assertAcked( + prepareCreate(INDEX_NAME).setMapping(mapping) + .addAlias(new Alias("alias")) + .setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.analysis.analyzer.tv_test.tokenizer", "standard") + .put("index.refresh_interval", -1) + .putList("index.analysis.analyzer.tv_test.filter", "lowercase") + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + ) + ); + final String replica = internalCluster().startDataOnlyNode(); + ensureGreen(INDEX_NAME); + final String id = routingKeyForShard(INDEX_NAME, 0); + final String routingOtherShard = routingKeyForShard(INDEX_NAME, 1); + + // index doc 1 + client().prepareIndex(INDEX_NAME) + .setId(Integer.toString(1)) + .setSource(jsonBuilder().startObject().field("field", "the quick brown fox jumps over the lazy dog").endObject()) + .setRouting(id) + .execute() + .actionGet(); + + // non realtime termvectors 1 + TermVectorsResponse response = client().prepareTermVectors(indexOrAlias(), Integer.toString(1)).setRealtime(false).get(); + assertFalse(response.isExists()); + + // realtime termvectors (preference = _replica) + TermVectorsRequestBuilder resp = client(replica).prepareTermVectors(indexOrAlias(), Integer.toString(1)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setPreference(Preference.REPLICA.type()) + .setRealtime(true) + .setSelectedFields(); + response = resp.execute().actionGet(); + + assertFalse(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + + // realtime termvectors (with routing set) + resp = client(replica).prepareTermVectors(indexOrAlias(), Integer.toString(1)) + .setPayloads(true) + .setOffsets(true) + .setPositions(true) + .setRouting(routingOtherShard) + .setSelectedFields(); + response = resp.execute().actionGet(); + + assertFalse(response.isExists()); + assertThat(response.getIndex(), equalTo(INDEX_NAME)); + + } + } diff --git a/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java index a76506b39f811..b7e8a29bd4027 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TransportTermVectorsAction.java @@ -38,6 +38,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.routing.GroupShardsIterator; +import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; @@ -87,15 +88,24 @@ public TransportTermVectorsAction( @Override protected ShardIterator shards(ClusterState state, InternalRequest request) { + + String preference = request.request().preference; + // For a real time request on a seg rep index, use primary shard as the preferred query shard. + if (request.request().realtime() + && preference == null + && state.getMetadata().isSegmentReplicationEnabled(request.concreteIndex())) { + preference = Preference.PRIMARY.type(); + } + if (request.request().doc() != null && request.request().routing() == null) { // artificial document without routing specified, ignore its "id" and use either random shard or according to preference GroupShardsIterator groupShardsIter = clusterService.operationRouting() - .searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference()); + .searchShards(state, new String[] { request.concreteIndex() }, null, preference); return groupShardsIter.iterator().next(); } return clusterService.operationRouting() - .getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()); + .getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), preference); } @Override From 4e1022e2e85ce16c8772e76c1e1d857e89b6c08c Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Fri, 8 Sep 2023 23:10:11 +0530 Subject: [PATCH 004/388] Add Tracing Instrumentation at Network and Rest layer (#9415) * Add Instrumentation Signed-off-by: Gagan Juneja * update changelog Signed-off-by: Gagan Juneja * clear the context Signed-off-by: Gagan Juneja * Add Instrumentation Signed-off-by: Gagan Juneja * Fix test cases Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Add Instrumentation Signed-off-by: Gagan Juneja * Add Instrumentation Signed-off-by: Gagan Juneja * Remove internal transport action check Signed-off-by: Gagan Juneja * Changes Signed-off-by: Gagan Juneja * Refactor code Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Fix failing test cases Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Fix failing test cases Signed-off-by: Gagan Juneja * Fix failing test cases Signed-off-by: Gagan Juneja * Fix failing test cases Signed-off-by: Gagan Juneja * Fix failing test cases Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Fix failing test cases Signed-off-by: Gagan Juneja * Fix java doc Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address merge conflicts Signed-off-by: Gagan Juneja * Address merge conflicts Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Enable NumberOfTraceIDsEqualToRequests test Signed-off-by: Gagan Juneja * Fix java doc Signed-off-by: Gagan Juneja * Fix java doc Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Fix test case Signed-off-by: Gagan Juneja * Remove instrumentation from TransportService Signed-off-by: Gagan Juneja --------- Signed-off-by: Gagan Juneja Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- CHANGELOG.md | 1 + .../telemetry/tracing/DefaultTracer.java | 8 +- .../telemetry/tracing/http/HttpTracer.java | 7 +- .../telemetry/tracing/noop/NoopTracer.java | 2 +- .../netty4/Netty4HttpServerTransport.java | 6 +- .../transport/Netty4ModulePlugin.java | 7 +- .../http/netty4/Netty4BadRequestTests.java | 4 +- .../Netty4HttpServerPipeliningTests.java | 4 +- .../Netty4HttpServerTransportTests.java | 22 ++-- .../discovery/ec2/Ec2DiscoveryTests.java | 10 +- .../discovery/ec2/Ec2RetriesTests.java | 4 +- .../discovery/gce/GceDiscoveryTests.java | 3 +- .../InMemorySingletonSpanExporter.java | 26 +++- .../TelemetryTracerDisabledSanityIT.java | 3 +- .../TelemetryTracerEnabledSanityIT.java | 31 +++-- .../http/nio/NioHttpServerTransport.java | 6 +- .../transport/nio/NioTransportPlugin.java | 7 +- .../http/nio/NioHttpServerTransportTests.java | 22 ++-- ...rossClusterSearchUnavailableClusterIT.java | 3 +- .../common/network/NetworkModule.java | 7 +- .../http/AbstractHttpServerTransport.java | 39 ++++-- .../main/java/org/opensearch/node/Node.java | 36 +++--- .../org/opensearch/plugins/NetworkPlugin.java | 4 +- .../telemetry/tracing/SpanBuilder.java | 4 +- ...hreadContextBasedTracerContextStorage.java | 2 +- .../telemetry/tracing/WrappedTracer.java | 4 +- .../channels/TraceableHttpChannel.java | 88 +++++++++++++ .../channels/TraceableRestChannel.java | 106 ++++++++++++++++ .../tracing/channels/package-info.java | 12 ++ .../listener/TraceableActionListener.java | 77 ++++++++++++ .../tracing/listener/package-info.java | 12 ++ .../transport/TransportService.java | 13 +- ...tAddVotingConfigExclusionsActionTests.java | 4 +- ...learVotingConfigExclusionsActionTests.java | 4 +- .../node/tasks/TaskManagerTestCase.java | 4 +- .../TransportRemoteStoreStatsActionTests.java | 4 +- ...portVerifyShardBeforeCloseActionTests.java | 4 +- .../indices/get/GetIndexActionTests.java | 4 +- .../settings/get/GetSettingsActionTests.java | 4 +- .../action/bulk/TransportBulkActionTests.java | 4 +- .../bulk/TransportBulkActionTookTests.java | 4 +- .../get/TransportMultiGetActionTests.java | 4 +- .../action/main/MainActionTests.java | 4 +- ...TransportResyncReplicationActionTests.java | 4 +- .../search/CreatePitControllerTests.java | 9 +- .../search/MultiSearchActionTookTests.java | 4 +- .../search/TransportDeletePitActionTests.java | 17 +-- .../TransportMultiSearchActionTests.java | 7 +- .../search/TransportSearchActionTests.java | 22 +++- .../TransportBroadcastByNodeActionTests.java | 4 +- ...ransportClusterManagerNodeActionTests.java | 4 +- .../nodes/TransportNodesActionTests.java | 4 +- .../BroadcastReplicationTests.java | 4 +- .../TransportReplicationActionTests.java | 7 +- ...ReplicationAllPermitsAcquisitionTests.java | 4 +- ...rtWriteActionForIndexingPressureTests.java | 4 +- .../TransportWriteActionTests.java | 10 +- ...ortInstanceSingleOperationActionTests.java | 4 +- .../TransportMultiTermVectorsActionTests.java | 4 +- .../cluster/NodeConnectionsServiceTests.java | 4 +- .../action/shard/ShardStateActionTests.java | 4 +- ...ransportGetWeightedRoutingActionTests.java | 5 +- ...ClusterAwarenessAttributesHealthTests.java | 4 +- .../ClusterAwarenessHealthTests.java | 4 +- ...BootstrapServiceDeprecatedMasterTests.java | 4 +- .../ClusterBootstrapServiceTests.java | 4 +- .../coordination/FollowersCheckerTests.java | 19 ++- .../cluster/coordination/JoinHelperTests.java | 13 +- .../coordination/LeaderCheckerTests.java | 13 +- .../cluster/coordination/NodeJoinTests.java | 4 +- .../coordination/PreVoteCollectorTests.java | 4 +- .../PublicationTransportHandlerTests.java | 4 +- .../DecommissionControllerTests.java | 4 +- .../DecommissionServiceTests.java | 4 +- .../health/ClusterStateHealthTests.java | 4 +- .../routing/WeightedRoutingServiceTests.java | 5 +- .../common/network/NetworkModuleTests.java | 14 ++- .../discovery/DiscoveryModuleTests.java | 3 +- .../FileBasedSeedHostsProviderTests.java | 4 +- ...shakingTransportAddressConnectorTests.java | 4 +- .../opensearch/discovery/PeerFinderTests.java | 4 +- .../discovery/SeedHostsResolverTests.java | 16 ++- .../extensions/ExtensionsManagerTests.java | 7 +- ...ExtensionTransportActionsHandlerTests.java | 4 +- .../RestInitializeExtensionActionTests.java | 4 +- .../rest/RestSendToExtensionActionTests.java | 4 +- .../AbstractHttpServerTransportTests.java | 7 +- .../opensearch/index/IndexModuleTests.java | 4 +- .../GlobalCheckpointSyncActionTests.java | 4 +- ...tentionLeaseBackgroundSyncActionTests.java | 4 +- .../seqno/RetentionLeaseSyncActionTests.java | 4 +- .../indices/cluster/ClusterStateChanges.java | 4 +- ...ClusterStateServiceRandomUpdatesTests.java | 4 +- .../PrimaryShardReplicationSourceTests.java | 4 +- .../SegmentReplicationSourceServiceTests.java | 4 +- .../SegmentReplicationTargetServiceTests.java | 4 +- .../PublishCheckpointActionTests.java | 4 +- .../RepositoriesServiceTests.java | 4 +- .../SearchBackpressureServiceTests.java | 3 +- .../snapshots/SnapshotResiliencyTests.java | 3 +- ...askCancellationMonitoringServiceTests.java | 3 +- .../ProxyConnectionStrategyTests.java | 66 ++++++++-- .../RemoteClusterAwareClientTests.java | 19 ++- .../transport/RemoteClusterClientTests.java | 23 +++- .../RemoteClusterConnectionTests.java | 66 ++++++++-- .../transport/RemoteClusterServiceTests.java | 58 +++++++-- .../SniffConnectionStrategyTests.java | 116 +++++++++++++++--- .../transport/TransportActionProxyTests.java | 3 +- ...ortServiceDeserializationFailureTests.java | 4 +- .../TransportServiceHandshakeTests.java | 4 +- .../AbstractCoordinatorTestCase.java | 4 +- .../java/org/opensearch/node/MockNode.java | 26 +++- .../test/OpenSearchIntegTestCase.java | 6 +- .../disruption/DisruptableMockTransport.java | 6 +- .../test/transport/MockTransport.java | 7 +- .../test/transport/MockTransportService.java | 38 +++--- .../AbstractSimpleTransportTestCase.java | 4 +- .../DisruptableMockTransportTests.java | 7 +- .../test/telemetry/tracing/MockSpan.java | 8 ++ .../test/telemetry/tracing/MockSpanData.java | 24 +++- .../tracing/StrictCheckSpanProcessor.java | 3 +- .../NumberOfTraceIDsEqualToRequests.java | 28 ++++- 122 files changed, 1207 insertions(+), 276 deletions(-) create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/channels/package-info.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/listener/package-info.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d6b4660d85d49..5abbeee490a3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -188,6 +188,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Cleanup Unreferenced file on segment merge failure ([#9503](https://github.com/opensearch-project/OpenSearch/pull/9503)) - Move ZStd to a plugin ([#9658](https://github.com/opensearch-project/OpenSearch/pull/9658)) - [Remote Store] Add support for Remote Translog Store upload stats in `_nodes/stats/` API ([#8908](https://github.com/opensearch-project/OpenSearch/pull/8908)) +- Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) - Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916)) - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java index 009cc673058d3..5b7795a5647f5 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java @@ -119,9 +119,13 @@ protected void addDefaultAttributes(Span span) { } @Override - public Span startSpan(String spanName, Map> headers, Attributes attributes) { + public Span startSpan(SpanCreationContext spanCreationContext, Map> headers) { Optional propagatedSpan = tracingTelemetry.getContextPropagator().extractFromHeaders(headers); - return startSpan(spanName, propagatedSpan.map(SpanContext::new).orElse(null), attributes); + return startSpan( + spanCreationContext.getSpanName(), + propagatedSpan.map(SpanContext::new).orElse(null), + spanCreationContext.getAttributes() + ); } } diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java index cb2bf73775564..b0692f1b62a48 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java @@ -10,7 +10,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.telemetry.tracing.Span; -import org.opensearch.telemetry.tracing.attributes.Attributes; +import org.opensearch.telemetry.tracing.SpanCreationContext; import java.util.List; import java.util.Map; @@ -28,10 +28,9 @@ public interface HttpTracer { /** * Start the span with propagating the tracing info from the HttpRequest header. * - * @param spanName span name. + * @param spanCreationContext span name. * @param header http request header. - * @param attributes span attributes. * @return span. */ - Span startSpan(String spanName, Map> header, Attributes attributes); + Span startSpan(SpanCreationContext spanCreationContext, Map> header); } diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java index 96a969369c3a9..d7206a3c6b094 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java @@ -81,7 +81,7 @@ public void close() { } @Override - public Span startSpan(String spanName, Map> header, Attributes attributes) { + public Span startSpan(SpanCreationContext spanCreationContext, Map> header) { return NoopSpan.INSTANCE; } } diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java index 775c8d4405cdb..0271472125814 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java @@ -52,6 +52,7 @@ import org.opensearch.http.HttpHandlingSettings; import org.opensearch.http.HttpReadTimeoutException; import org.opensearch.http.HttpServerChannel; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.NettyAllocator; import org.opensearch.transport.NettyByteBufSizer; @@ -191,9 +192,10 @@ public Netty4HttpServerTransport( NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, ClusterSettings clusterSettings, - SharedGroupFactory sharedGroupFactory + SharedGroupFactory sharedGroupFactory, + Tracer tracer ) { - super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings); + super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, tracer); Netty4Utils.setAvailableProcessors(OpenSearchExecutors.NODE_PROCESSORS_SETTING.get(settings)); NettyAllocator.logAllocatorDescriptionIfNeeded(); this.sharedGroupFactory = sharedGroupFactory; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java index fcf128e7c79e5..ca51d70702a82 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java @@ -48,6 +48,7 @@ import org.opensearch.http.netty4.Netty4HttpServerTransport; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.netty4.Netty4Transport; @@ -122,7 +123,8 @@ public Map> getHttpTransports( NamedXContentRegistry xContentRegistry, NetworkService networkService, HttpServerTransport.Dispatcher dispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { return Collections.singletonMap( NETTY_HTTP_TRANSPORT_NAME, @@ -134,7 +136,8 @@ public Map> getHttpTransports( xContentRegistry, dispatcher, clusterSettings, - getSharedGroupFactory(settings) + getSharedGroupFactory(settings), + tracer ) ); } diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java index ea36d1ee26852..03990c173d547 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java @@ -47,6 +47,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -112,7 +113,8 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, xContentRegistry(), dispatcher, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - new SharedGroupFactory(Settings.EMPTY) + new SharedGroupFactory(Settings.EMPTY), + NoopTracer.INSTANCE ) ) { httpServerTransport.start(); diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java index 4201a4d0a8b4b..af868a3a3cb88 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java @@ -45,6 +45,7 @@ import org.opensearch.http.HttpResponse; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.NullDispatcher; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -135,7 +136,8 @@ class CustomNettyHttpServerTransport extends Netty4HttpServerTransport { xContentRegistry(), new NullDispatcher(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ); } diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java index adcada811e537..d892918decfb5 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java @@ -55,6 +55,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.opensearch.threadpool.TestThreadPool; @@ -198,7 +199,8 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, xContentRegistry(), dispatcher, clusterSettings, - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -247,7 +249,8 @@ public void testBindUnavailableAddress() { xContentRegistry(), new NullDispatcher(), clusterSettings, - new SharedGroupFactory(Settings.EMPTY) + new SharedGroupFactory(Settings.EMPTY), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -265,7 +268,8 @@ public void testBindUnavailableAddress() { xContentRegistry(), new NullDispatcher(), clusterSettings, - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ) ) { BindHttpException bindHttpException = expectThrows(BindHttpException.class, otherTransport::start); @@ -317,7 +321,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, clusterSettings, - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -379,7 +384,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, clusterSettings, - new SharedGroupFactory(Settings.EMPTY) + new SharedGroupFactory(Settings.EMPTY), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -448,7 +454,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -521,7 +528,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ) ) { transport.start(); diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java index 23070c389a7b1..6bed6564cfd36 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java @@ -48,6 +48,7 @@ import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.Transport; import org.opensearch.transport.TransportService; @@ -99,7 +100,14 @@ public TransportAddress[] addressesFromString(String address) { return new TransportAddress[] { poorMansDNS.getOrDefault(address, buildNewFakeTransportAddress()) }; } }; - return new MockTransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null); + return new MockTransportService( + Settings.EMPTY, + transport, + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + null, + NoopTracer.INSTANCE + ); } protected List buildDynamicHosts(Settings nodeSettings, int nodes) { diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java index c792fe6d96728..3311ddc8842f2 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java @@ -47,6 +47,7 @@ import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.discovery.SeedHostsProvider; import org.opensearch.discovery.SeedHostsResolver; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.TransportService; import org.opensearch.transport.nio.MockNioTransport; @@ -80,7 +81,8 @@ protected MockTransportService createTransportService() { ), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - null + null, + NoopTracer.INSTANCE ); } diff --git a/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java index c63085deb466f..b4af9773f33de 100644 --- a/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java +++ b/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java @@ -38,6 +38,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -109,7 +110,7 @@ public void setProjectName() { @Before public void createTransportService() { - transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null); + transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, NoopTracer.INSTANCE); } @After diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/InMemorySingletonSpanExporter.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/InMemorySingletonSpanExporter.java index 5fb6acc8346db..6dd451ea37465 100644 --- a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/InMemorySingletonSpanExporter.java +++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/InMemorySingletonSpanExporter.java @@ -11,7 +11,9 @@ import org.opensearch.test.telemetry.tracing.MockSpanData; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import io.opentelemetry.sdk.common.CompletableResultCode; @@ -21,7 +23,7 @@ public class InMemorySingletonSpanExporter implements SpanExporter { - private static final InMemorySingletonSpanExporter INSTANCE = new InMemorySingletonSpanExporter(InMemorySpanExporter.create()); + public static final InMemorySingletonSpanExporter INSTANCE = new InMemorySingletonSpanExporter(InMemorySpanExporter.create()); private static InMemorySpanExporter delegate; @@ -62,10 +64,30 @@ private List convertSpanDataListToMockSpanDataList(List spanData.getStartEpochNanos(), spanData.getEndEpochNanos(), spanData.hasEnded(), - spanData.getName() + spanData.getName(), + getAttributes(spanData) ) ) .collect(Collectors.toList()); return mockSpanDataList; } + + private Map getAttributes(SpanData spanData) { + if (spanData.getAttributes() != null) { + return spanData.getAttributes() + .asMap() + .entrySet() + .stream() + .collect(Collectors.toMap(e -> e.getKey().getKey(), e -> e.getValue())); + } else { + return Collections.emptyMap(); + } + } + + /** + * Clears the state. + */ + public void reset() { + delegate.reset(); + } } diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java index 476a5a9cabdc7..949a58f6cab41 100644 --- a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java +++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java @@ -63,6 +63,8 @@ public void testSanityCheckWhenTracingDisabled() throws Exception { ensureGreen(); refresh(); + InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.INSTANCE; + exporter.reset(); // Make the search call; client.prepareSearch().setQuery(queryStringQuery("fox")).get(); @@ -70,7 +72,6 @@ public void testSanityCheckWhenTracingDisabled() throws Exception { // Sleep for about 3s to wait for traces are published (the delay is 1s) Thread.sleep(3000); - InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.create(); assertTrue(exporter.getFinishedSpanItems().isEmpty()); } diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java index 9f99099e85c9f..2d0111e64faad 100644 --- a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java +++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java @@ -14,6 +14,7 @@ import org.opensearch.plugins.Plugin; import org.opensearch.telemetry.OTelTelemetrySettings; import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.tracing.attributes.Attributes; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.telemetry.tracing.TelemetryValidators; import org.opensearch.test.telemetry.tracing.validators.AllSpansAreEndedProperly; @@ -38,6 +39,7 @@ protected Settings nodeSettings(int nodeOrdinal) { "org.opensearch.telemetry.tracing.InMemorySingletonSpanExporter" ) .put(OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING.getKey(), TimeValue.timeValueSeconds(1)) + .put(TelemetrySettings.TRACER_SAMPLER_PROBABILITY.getKey(), 1.0d) .build(); } @@ -52,13 +54,9 @@ protected boolean addMockTelemetryPlugin() { } public void testSanityChecksWhenTracingEnabled() throws Exception { - Client client = client(); + Client client = internalCluster().clusterManagerClient(); // ENABLE TRACING - client.admin() - .cluster() - .prepareUpdateSettings() - .setTransientSettings(Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true)) - .get(); + updateTelemetrySetting(client, true); // Create Index and ingest data String indexName = "test-index-11"; @@ -70,9 +68,12 @@ public void testSanityChecksWhenTracingEnabled() throws Exception { ensureGreen(); refresh(); - // Make the search calls; - client.prepareSearch().setQuery(queryStringQuery("fox")).get(); - client.prepareSearch().setQuery(queryStringQuery("jumps")).get(); + // Make the search calls; adding the searchType and PreFilterShardSize to make the query path predictable across all the runs. + client.prepareSearch().setSearchType("query_then_fetch").setPreFilterShardSize(3).setQuery(queryStringQuery("fox")).get(); + client.prepareSearch().setSearchType("query_then_fetch").setPreFilterShardSize(3).setQuery(queryStringQuery("jumps")).get(); + + ensureGreen(); + refresh(); // Sleep for about 3s to wait for traces are published, delay is (the delay is 1s). Thread.sleep(3000); @@ -81,15 +82,23 @@ public void testSanityChecksWhenTracingEnabled() throws Exception { Arrays.asList( new AllSpansAreEndedProperly(), new AllSpansHaveUniqueId(), - new NumberOfTraceIDsEqualToRequests(), + new NumberOfTraceIDsEqualToRequests(Attributes.create().addAttribute("action", "indices:data/read/search[phase/query]")), new TotalRootSpansEqualToRequests() ) ); - InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.create(); + InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.INSTANCE; if (!exporter.getFinishedSpanItems().isEmpty()) { validators.validate(exporter.getFinishedSpanItems(), 2); } } + private static void updateTelemetrySetting(Client client, boolean value) { + client.admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), value)) + .get(); + } + } diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java index 6165df6a591d6..ecf9ad9f17f87 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java @@ -56,6 +56,7 @@ import org.opensearch.nio.NioSocketChannel; import org.opensearch.nio.ServerChannelContext; import org.opensearch.nio.SocketChannelContext; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.nio.NioGroupFactory; import org.opensearch.transport.nio.PageAllocator; @@ -106,9 +107,10 @@ public NioHttpServerTransport( NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, NioGroupFactory nioGroupFactory, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { - super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings); + super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, tracer); this.pageAllocator = new PageAllocator(pageCacheRecycler); this.nioGroupFactory = nioGroupFactory; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java index a3475c2ea2969..ec266d76eff3d 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java @@ -50,6 +50,7 @@ import org.opensearch.http.nio.NioHttpServerTransport; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -117,7 +118,8 @@ public Map> getHttpTransports( NamedXContentRegistry xContentRegistry, NetworkService networkService, HttpServerTransport.Dispatcher dispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { return Collections.singletonMap( NIO_HTTP_TRANSPORT_NAME, @@ -130,7 +132,8 @@ public Map> getHttpTransports( xContentRegistry, dispatcher, getNioGroupFactory(settings), - clusterSettings + clusterSettings, + tracer ) ); } diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java index 89e6e9ce88408..09594673de5b2 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java @@ -56,6 +56,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.opensearch.threadpool.TestThreadPool; @@ -186,7 +187,8 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -236,7 +238,8 @@ public void testBindUnavailableAddress() { xContentRegistry(), new NullDispatcher(), new NioGroupFactory(Settings.EMPTY, logger), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -255,7 +258,8 @@ public void testBindUnavailableAddress() { xContentRegistry(), new NullDispatcher(), new NioGroupFactory(Settings.EMPTY, logger), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) ) { BindHttpException bindHttpException = expectThrows(BindHttpException.class, () -> otherTransport.start()); @@ -298,7 +302,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -372,7 +377,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, new NioGroupFactory(Settings.EMPTY, logger), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -438,7 +444,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) ) { transport.start(); @@ -500,7 +507,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th xContentRegistry(), dispatcher, new NioGroupFactory(settings, logger), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) ) { transport.start(); diff --git a/qa/ccs-unavailable-clusters/src/test/java/org/opensearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/test/java/org/opensearch/search/CrossClusterSearchUnavailableClusterIT.java index 5d2f835aff4b0..ea20d8000f640 100644 --- a/qa/ccs-unavailable-clusters/src/test/java/org/opensearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/test/java/org/opensearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -67,6 +67,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.internal.InternalSearchResponse; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.rest.OpenSearchRestTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -118,7 +119,7 @@ private static MockTransportService startTransport( boolean success = false; final Settings s = Settings.builder().put("node.name", id).build(); ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s); - MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, null); + MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, NoopTracer.INSTANCE); try { newService.registerRequestHandler(ClusterSearchShardsAction.NAME, ThreadPool.Names.SAME, ClusterSearchShardsRequest::new, (request, channel, task) -> { diff --git a/server/src/main/java/org/opensearch/common/network/NetworkModule.java b/server/src/main/java/org/opensearch/common/network/NetworkModule.java index 3539ea7f3f526..8870e26c373e9 100644 --- a/server/src/main/java/org/opensearch/common/network/NetworkModule.java +++ b/server/src/main/java/org/opensearch/common/network/NetworkModule.java @@ -57,6 +57,7 @@ import org.opensearch.plugins.NetworkPlugin; import org.opensearch.tasks.RawTaskStatus; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; import org.opensearch.transport.TransportInterceptor; @@ -147,7 +148,8 @@ public NetworkModule( NamedXContentRegistry xContentRegistry, NetworkService networkService, HttpServerTransport.Dispatcher dispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { this.settings = settings; for (NetworkPlugin plugin : plugins) { @@ -160,7 +162,8 @@ public NetworkModule( xContentRegistry, networkService, dispatcher, - clusterSettings + clusterSettings, + tracer ); for (Map.Entry> entry : httpTransportFactory.entrySet()) { registerHttpTransport(entry.getKey(), entry.getValue()); diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index ce02dfb21c587..ed44102d0abe4 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -54,6 +54,12 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanBuilder; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.telemetry.tracing.channels.TraceableHttpChannel; +import org.opensearch.telemetry.tracing.channels.TraceableRestChannel; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BindTransportException; @@ -105,7 +111,8 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo private final Set httpChannels = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set httpServerChannels = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final HttpTracer tracer; + private final HttpTracer httpTracer; + private final Tracer tracer; protected AbstractHttpServerTransport( Settings settings, @@ -114,7 +121,8 @@ protected AbstractHttpServerTransport( ThreadPool threadPool, NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer telemetryTracer ) { this.settings = settings; this.networkService = networkService; @@ -138,7 +146,8 @@ protected AbstractHttpServerTransport( this.port = SETTING_HTTP_PORT.get(settings); this.maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings); - this.tracer = new HttpTracer(settings, clusterSettings); + this.httpTracer = new HttpTracer(settings, clusterSettings); + this.tracer = telemetryTracer; } @Override @@ -352,19 +361,31 @@ protected void serverAcceptedChannel(HttpChannel httpChannel) { * @param httpChannel that received the http request */ public void incomingRequest(final HttpRequest httpRequest, final HttpChannel httpChannel) { - handleIncomingRequest(httpRequest, httpChannel, httpRequest.getInboundException()); + final Span span = tracer.startSpan(SpanBuilder.from(httpRequest), httpRequest.getHeaders()); + try (final SpanScope httpRequestSpanScope = tracer.withSpanInScope(span)) { + HttpChannel traceableHttpChannel = TraceableHttpChannel.create(httpChannel, span, tracer); + handleIncomingRequest(httpRequest, traceableHttpChannel, httpRequest.getInboundException()); + } } // Visible for testing void dispatchRequest(final RestRequest restRequest, final RestChannel channel, final Throwable badRequestCause) { + RestChannel traceableRestChannel = channel; final ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { - if (badRequestCause != null) { - dispatcher.dispatchBadRequest(channel, threadContext, badRequestCause); - } else { - dispatcher.dispatchRequest(restRequest, channel, threadContext); + final Span span = tracer.startSpan(SpanBuilder.from(restRequest)); + try (final SpanScope spanScope = tracer.withSpanInScope(span)) { + if (channel != null) { + traceableRestChannel = TraceableRestChannel.create(channel, span, tracer); + } + if (badRequestCause != null) { + dispatcher.dispatchBadRequest(traceableRestChannel, threadContext, badRequestCause); + } else { + dispatcher.dispatchRequest(restRequest, traceableRestChannel, threadContext); + } } } + } private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChannel httpChannel, final Exception exception) { @@ -401,7 +422,7 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan restRequest = innerRestRequest; } - final HttpTracer trace = tracer.maybeTraceRequest(restRequest, exception); + final HttpTracer trace = httpTracer.maybeTraceRequest(restRequest, exception); /* * We now want to create a channel used to send the response on. However, creating this channel can fail if there are invalid diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 2c987432199a0..1f8f17f8e8d91 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -588,6 +588,19 @@ protected Node( new ConsistentSettingsService(settings, clusterService, consistentSettings).newHashPublisher() ); } + + TracerFactory tracerFactory; + if (FeatureFlags.isEnabled(TELEMETRY)) { + final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings()); + List telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class); + TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, telemetrySettings); + tracerFactory = new TracerFactory(telemetrySettings, telemetryModule.getTelemetry(), threadPool.getThreadContext()); + } else { + tracerFactory = new NoopTracerFactory(); + } + + tracer = tracerFactory.getTracer(); + resourcesToClose.add(tracer::close); final IngestService ingestService = new IngestService( clusterService, threadPool, @@ -858,7 +871,8 @@ protected Node( xContentRegistry, networkService, restController, - clusterService.getClusterSettings() + clusterService.getClusterSettings(), + tracer ); Collection>> indexTemplateMetadataUpgraders = pluginsService.filterPlugins( Plugin.class @@ -888,7 +902,8 @@ protected Node( networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings(), - taskHeaders + taskHeaders, + tracer ); TopNSearchTasksLogger taskConsumer = new TopNSearchTasksLogger(settings, settingsModule.getClusterSettings()); transportService.getTaskManager().registerTaskResourceConsumer(taskConsumer); @@ -1076,18 +1091,6 @@ protected Node( searchModule.getIndexSearcherExecutor(threadPool) ); - TracerFactory tracerFactory; - if (FeatureFlags.isEnabled(TELEMETRY)) { - final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings()); - List telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class); - TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, telemetrySettings); - tracerFactory = new TracerFactory(telemetrySettings, telemetryModule.getTelemetry(), threadPool.getThreadContext()); - } else { - tracerFactory = new NoopTracerFactory(); - } - tracer = tracerFactory.getTracer(); - resourcesToClose.add(tracer::close); - final List> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) .stream() .map( @@ -1250,9 +1253,10 @@ protected TransportService newTransportService( TransportInterceptor interceptor, Function localNodeFactory, ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { - return new TransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); + return new TransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, tracer); } protected void processRecoverySettings(ClusterSettings clusterSettings, RecoverySettings recoverySettings) { diff --git a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java index de0aecc7833c9..f2f8e84f04e02 100644 --- a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java @@ -41,6 +41,7 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.http.HttpServerTransport; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; import org.opensearch.transport.TransportInterceptor; @@ -100,7 +101,8 @@ default Map> getHttpTransports( NamedXContentRegistry xContentRegistry, NetworkService networkService, HttpServerTransport.Dispatcher dispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { return Collections.emptyMap(); } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java index 80ac811591198..d247d48389fc8 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java @@ -82,7 +82,9 @@ private static Attributes buildSpanAttributes(HttpRequest httpRequest) { private static void populateHeader(HttpRequest httpRequest, Attributes attributes) { HEADERS_TO_BE_ADDED_AS_ATTRIBUTES.forEach(x -> { - if (httpRequest.getHeaders() != null && httpRequest.getHeaders().get(x) != null) { + if (httpRequest.getHeaders() != null + && httpRequest.getHeaders().get(x) != null + && (httpRequest.getHeaders().get(x).isEmpty() == false)) { attributes.addAttribute(x, Strings.collectionToCommaDelimitedString(httpRequest.getHeaders().get(x))); } }); diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java b/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java index d631d6ac01dd0..208df90f65d74 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/ThreadContextBasedTracerContextStorage.java @@ -70,7 +70,7 @@ public Map headers(Map source) { if (source.containsKey(CURRENT_SPAN)) { final SpanReference current = (SpanReference) source.get(CURRENT_SPAN); - if (current != null) { + if (current != null && current.getSpan() != null) { tracingTelemetry.getContextPropagator().inject(current.getSpan(), (key, value) -> headers.put(key, value)); } } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java index b58bbf08f9f9b..d8cf39d4a4d09 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java @@ -92,7 +92,7 @@ Tracer getDelegateTracer() { } @Override - public Span startSpan(String spanName, Map> headers, Attributes attributes) { - return defaultTracer.startSpan(spanName, headers, attributes); + public Span startSpan(SpanCreationContext spanCreationContext, Map> headers) { + return defaultTracer.startSpan(spanCreationContext, headers); } } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java new file mode 100644 index 0000000000000..9229d334dea01 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.channels; + +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.http.HttpChannel; +import org.opensearch.http.HttpResponse; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.telemetry.tracing.listener.TraceableActionListener; + +import java.net.InetSocketAddress; +import java.util.Objects; + +/** + * Tracer wrapped {@link HttpChannel} + */ +public class TraceableHttpChannel implements HttpChannel { + private final HttpChannel delegate; + private final Span span; + private final Tracer tracer; + + /** + * Constructor. + * + * @param delegate delegate + * @param span span + * @param tracer tracer + */ + private TraceableHttpChannel(HttpChannel delegate, Span span, Tracer tracer) { + this.span = Objects.requireNonNull(span); + this.delegate = Objects.requireNonNull(delegate); + this.tracer = Objects.requireNonNull(tracer); + } + + /** + * Factory method. + * + * @param delegate delegate + * @param span span + * @param tracer tracer + * @return http channel + */ + public static HttpChannel create(HttpChannel delegate, Span span, Tracer tracer) { + if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) { + return new TraceableHttpChannel(delegate, span, tracer); + } else { + return delegate; + } + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public void addCloseListener(ActionListener listener) { + delegate.addCloseListener(listener); + } + + @Override + public boolean isOpen() { + return delegate.isOpen(); + } + + @Override + public void sendResponse(HttpResponse response, ActionListener listener) { + delegate.sendResponse(response, TraceableActionListener.create(listener, span, tracer)); + } + + @Override + public InetSocketAddress getLocalAddress() { + return delegate.getLocalAddress(); + } + + @Override + public InetSocketAddress getRemoteAddress() { + return delegate.getRemoteAddress(); + } +} diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java new file mode 100644 index 0000000000000..d256c9d4d0e53 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.channels; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; + +import java.io.IOException; +import java.util.Objects; + +/** + * Tracer wrapped {@link RestChannel} + */ +public class TraceableRestChannel implements RestChannel { + + private final RestChannel delegate; + private final Span span; + + private final Tracer tracer; + + /** + * Constructor. + * + * @param delegate delegate + * @param span span + * @param tracer tracer + */ + private TraceableRestChannel(RestChannel delegate, Span span, Tracer tracer) { + this.span = Objects.requireNonNull(span); + this.delegate = Objects.requireNonNull(delegate); + this.tracer = Objects.requireNonNull(tracer); + } + + /** + * Factory method. + * @param delegate delegate + * @param span span + * @param tracer tracer + * @return rest channel + */ + public static RestChannel create(RestChannel delegate, Span span, Tracer tracer) { + if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) { + return new TraceableRestChannel(delegate, span, tracer); + } else { + return delegate; + } + } + + @Override + public XContentBuilder newBuilder() throws IOException { + return delegate.newBuilder(); + } + + @Override + public XContentBuilder newErrorBuilder() throws IOException { + return delegate.newErrorBuilder(); + } + + @Override + public XContentBuilder newBuilder(MediaType mediaType, boolean useFiltering) throws IOException { + return delegate.newBuilder(mediaType, useFiltering); + } + + @Override + public XContentBuilder newBuilder(MediaType mediaType, MediaType responseContentType, boolean useFiltering) throws IOException { + return delegate.newBuilder(mediaType, responseContentType, useFiltering); + } + + @Override + public BytesStreamOutput bytesOutput() { + return delegate.bytesOutput(); + } + + @Override + public RestRequest request() { + return delegate.request(); + } + + @Override + public boolean detailedErrorsEnabled() { + return delegate.detailedErrorsEnabled(); + } + + @Override + public void sendResponse(RestResponse response) { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.sendResponse(response); + } finally { + span.endSpan(); + } + } +} diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/package-info.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/package-info.java new file mode 100644 index 0000000000000..ee4b675d5dc30 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for tracing requests. + */ +package org.opensearch.telemetry.tracing.channels; diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java b/server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java new file mode 100644 index 0000000000000..3e201641a529b --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.listener; + +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; + +import java.util.Objects; + +/** + * Tracer wrapped {@link ActionListener} + * @param response. + */ +public class TraceableActionListener implements ActionListener { + + private final ActionListener delegate; + private final Span span; + private final Tracer tracer; + + /** + * Constructor. + * + * @param delegate delegate + * @param span span + * @param tracer tracer + */ + private TraceableActionListener(ActionListener delegate, Span span, Tracer tracer) { + this.delegate = Objects.requireNonNull(delegate); + this.span = Objects.requireNonNull(span); + this.tracer = Objects.requireNonNull(tracer); + } + + /** + * Factory method. + * @param delegate delegate + * @param span span + * @param tracer tracer + * @return action listener + */ + public static ActionListener create(ActionListener delegate, Span span, Tracer tracer) { + if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) { + return new TraceableActionListener(delegate, span, tracer); + } else { + return delegate; + } + } + + @Override + public void onResponse(Response response) { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.onResponse(response); + } finally { + span.endSpan(); + } + + } + + @Override + public void onFailure(Exception e) { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.onFailure(e); + } finally { + span.setError(e); + span.endSpan(); + } + + } +} diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/listener/package-info.java b/server/src/main/java/org/opensearch/telemetry/tracing/listener/package-info.java new file mode 100644 index 0000000000000..5dcb570c2bb2e --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/listener/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for tracing requests. + */ +package org.opensearch.telemetry.tracing.listener; diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index ff604d3435f21..52274872e8cc8 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -66,6 +66,7 @@ import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -132,11 +133,11 @@ protected boolean removeEldestEntry(Map.Entry eldest) { // tracer log private final Logger tracerLog; - volatile String[] tracerLogInclude; volatile String[] tracerLogExclude; private final RemoteClusterService remoteClusterService; + private final Tracer tracer; /** if set will call requests sent to this id to shortcut and executed locally */ volatile DiscoveryNode localNode = null; @@ -190,7 +191,8 @@ public TransportService( TransportInterceptor transportInterceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { this( settings, @@ -200,7 +202,8 @@ public TransportService( localNodeFactory, clusterSettings, taskHeaders, - new ClusterConnectionManager(settings, transport) + new ClusterConnectionManager(settings, transport), + tracer ); } @@ -212,7 +215,8 @@ public TransportService( Function localNodeFactory, @Nullable ClusterSettings clusterSettings, Set taskHeaders, - ConnectionManager connectionManager + ConnectionManager connectionManager, + Tracer tracer ) { this.transport = transport; transport.setSlowLogThreshold(TransportSettings.SLOW_OPERATION_THRESHOLD_SETTING.get(settings)); @@ -227,6 +231,7 @@ public TransportService( this.interceptor = transportInterceptor; this.asyncSender = interceptor.interceptSender(this::sendRequestInternal); this.remoteClusterClient = DiscoveryNode.isRemoteClusterClient(settings); + this.tracer = tracer; remoteClusterService = new RemoteClusterService(settings, this); responseHandlers = transport.getResponseHandlers(); if (clusterSettings != null) { diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 26b0c5ef05cdc..a015e671f4872 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -55,6 +55,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.TestThreadPool; @@ -139,7 +140,8 @@ public void setupForTest() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); final Settings.Builder nodeSettingsBuilder = Settings.builder(); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java index 535080b196f29..10e4ab6388be4 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java @@ -49,6 +49,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.TestThreadPool; @@ -111,7 +112,8 @@ public void setupForTest() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); new TransportClearVotingConfigExclusionsAction( diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index 6a508f5843b22..07e149dd72164 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -60,6 +60,7 @@ import org.opensearch.tasks.TaskCancellationService; import org.opensearch.tasks.TaskManager; import org.opensearch.tasks.TaskResourceTrackingService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.tasks.MockTaskManager; import org.opensearch.threadpool.RunnableTaskExecutionListener; @@ -216,7 +217,8 @@ public TestNode(String name, ThreadPool threadPool, Settings settings) { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddressDiscoveryNodeFunction, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ) { @Override protected TaskManager createTaskManager( diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index 1695e3dea59c6..ed73c2ef6ace5 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -31,6 +31,7 @@ import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.transport.MockTransport; import org.opensearch.transport.TransportService; @@ -85,7 +86,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); when(remoteStoreStatsTrackerFactory.getRemoteSegmentTransferTracker(any())).thenReturn(mock(RemoteSegmentTransferTracker.class)); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java index 371f326617b61..ef26bc225b0c7 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java @@ -62,6 +62,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ReplicationGroup; import org.opensearch.indices.IndicesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -139,7 +140,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java index 8f4e0e3b75e32..2d9ec2b6d3c02 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java @@ -46,6 +46,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; import org.opensearch.indices.IndicesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -85,7 +86,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java index d8e0f96292e27..f2b6688716e70 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java @@ -45,6 +45,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -108,7 +109,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java index 1229584fa99ac..098bd8e8d8cfe 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java @@ -62,6 +62,7 @@ import org.opensearch.index.VersionType; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.SystemIndices; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.opensearch.test.transport.CapturingTransport; @@ -153,7 +154,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java index b9dca5f2573f3..829eee45cac5b 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java @@ -57,6 +57,7 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.indices.SystemIndices; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.opensearch.test.transport.CapturingTransport; @@ -125,7 +126,8 @@ private TransportBulkAction createAction(boolean controlled, AtomicLong expected TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java b/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java index 0503bb39427a1..52443e695e014 100644 --- a/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java +++ b/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java @@ -62,6 +62,7 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -168,7 +169,8 @@ public static void beforeClass() throws Exception { randomBase64UUID() ), null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ) { @Override public TaskManager getTaskManager() { diff --git a/server/src/test/java/org/opensearch/action/main/MainActionTests.java b/server/src/test/java/org/opensearch/action/main/MainActionTests.java index fc9a83c10dbbc..b43dc2a80cd37 100644 --- a/server/src/test/java/org/opensearch/action/main/MainActionTests.java +++ b/server/src/test/java/org/opensearch/action/main/MainActionTests.java @@ -44,6 +44,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.Transport; import org.opensearch.transport.TransportService; @@ -109,7 +110,8 @@ public void testMainActionClusterAvailable() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); TransportMainAction action = new TransportMainAction(settings, transportService, mock(ActionFilters.class), clusterService); AtomicReference responseRef = new AtomicReference<>(); diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java index e88cc4c5b1d52..fbac465f946f4 100644 --- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java @@ -64,6 +64,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -145,7 +146,8 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception { NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java b/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java index 897c2a6198eac..2643aa5b6db01 100644 --- a/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java @@ -32,6 +32,7 @@ import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -180,7 +181,7 @@ public void testUpdatePitAfterCreatePitSuccess() throws InterruptedException { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -273,7 +274,7 @@ public void testUpdatePitAfterCreatePitFailure() throws InterruptedException { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -366,7 +367,7 @@ public void testUpdatePitFailureForNodeDrop() throws InterruptedException { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -462,7 +463,7 @@ public void testUpdatePitFailureWhereAllNodesDown() throws InterruptedException Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); diff --git a/server/src/test/java/org/opensearch/action/search/MultiSearchActionTookTests.java b/server/src/test/java/org/opensearch/action/search/MultiSearchActionTookTests.java index 173bee40d9ae8..94ba5b0a8768b 100644 --- a/server/src/test/java/org/opensearch/action/search/MultiSearchActionTookTests.java +++ b/server/src/test/java/org/opensearch/action/search/MultiSearchActionTookTests.java @@ -49,6 +49,7 @@ import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -153,7 +154,8 @@ private TransportMultiSearchAction createTransportMultiSearchAction(boolean cont TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ) { @Override public TaskManager getTaskManager() { diff --git a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java index 6713faf78a58c..8d3cdc070c695 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java @@ -27,6 +27,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.ThreadPool; @@ -141,7 +142,7 @@ public void testDeletePitSuccess() throws InterruptedException, ExecutionExcepti Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -204,7 +205,7 @@ public void testDeleteAllPITSuccess() throws InterruptedException, ExecutionExce Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -279,7 +280,7 @@ public void testDeletePitWhenNodeIsDown() throws InterruptedException, Execution Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -342,7 +343,7 @@ public void testDeletePitWhenAllNodesAreDown() { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -400,7 +401,7 @@ public void testDeletePitFailure() { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -465,7 +466,7 @@ public void testDeleteAllPitWhenNodeIsDown() { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -542,7 +543,7 @@ public void testDeleteAllPitWhenAllNodesAreDown() { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -615,7 +616,7 @@ public void testDeleteAllPitFailure() { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); diff --git a/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java index 6bb7401615b8f..48970e2b96add 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java @@ -51,6 +51,7 @@ import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -87,7 +88,8 @@ public void testParentTaskId() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ) { @Override public TaskManager getTaskManager() { @@ -151,7 +153,8 @@ public void testBatchExecute() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ) { @Override public TaskManager getTaskManager() { diff --git a/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java index e278f088508fc..c4bf8a5d87172 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java @@ -75,6 +75,7 @@ import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.search.internal.SearchContext; import org.opensearch.search.sort.SortBuilders; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -234,7 +235,14 @@ public void testMergeShardsIterators() { } public void testProcessRemoteShards() { - try (TransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + TransportService transportService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { RemoteClusterService service = transportService.getRemoteClusterService(); assertFalse(service.isCrossClusterSearchEnabled()); Map searchShardsResponseMap = new HashMap<>(); @@ -451,7 +459,9 @@ public void testCCSRemoteReduceMergeFails() throws Exception { OriginalIndices localIndices = local ? new OriginalIndices(new String[] { "index" }, SearchRequest.DEFAULT_INDICES_OPTIONS) : null; TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0); Function reduceContext = finalReduce -> null; - try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, NoopTracer.INSTANCE) + ) { service.start(); service.acceptIncomingRequests(); RemoteClusterService remoteClusterService = service.getRemoteClusterService(); @@ -507,7 +517,9 @@ public void testCCSRemoteReduce() throws Exception { OriginalIndices localIndices = local ? new OriginalIndices(new String[] { "index" }, SearchRequest.DEFAULT_INDICES_OPTIONS) : null; int totalClusters = numClusters + (local ? 1 : 0); TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0); - try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, NoopTracer.INSTANCE) + ) { service.start(); service.acceptIncomingRequests(); RemoteClusterService remoteClusterService = service.getRemoteClusterService(); @@ -748,7 +760,9 @@ public void testCollectSearchShards() throws Exception { Settings.Builder builder = Settings.builder(); MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder); Settings settings = builder.build(); - try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, NoopTracer.INSTANCE) + ) { service.start(); service.acceptIncomingRequests(); RemoteClusterService remoteClusterService = service.getRemoteClusterService(); diff --git a/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index 22a07f15fc5ed..4305151965ab6 100644 --- a/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -68,6 +68,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -237,7 +238,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java index 56a31f4d3cbd7..9ae1310a8b15c 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java @@ -63,6 +63,7 @@ import org.opensearch.discovery.ClusterManagerNotDiscoveredException; import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -117,7 +118,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java index 28f01d0e6ea4a..445934b0ccdfd 100644 --- a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java @@ -46,6 +46,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -209,7 +210,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java index 9022b32630d5a..77c9c64ad6611 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java @@ -62,6 +62,7 @@ import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -125,7 +126,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java index 6b717fe187078..0bee99f4d5656 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java @@ -87,6 +87,7 @@ import org.opensearch.indices.IndexClosedException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.cluster.ClusterStateChanges; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.test.transport.MockTransportService; @@ -195,7 +196,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -1325,7 +1327,8 @@ public void testRetryOnReplicaWithRealTransport() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index 088c2b0eb14f4..839ac7ab5bb92 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -65,6 +65,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.indices.IndicesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportException; @@ -232,7 +233,8 @@ public String executor() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, bta -> node1, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java index 06a13976756a9..4a2185d1558f7 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java @@ -37,6 +37,7 @@ import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -102,7 +103,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java index c6421bfa77e70..9d2069ac16190 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java @@ -65,6 +65,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; import org.opensearch.node.NodeClosedException; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; @@ -288,7 +289,8 @@ public void testReplicaProxy() throws InterruptedException, ExecutionException { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -407,7 +409,8 @@ public void testPrimaryClosedDoesNotFailShard() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -461,7 +464,8 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ), TransportWriteActionTests.this.clusterService, null, diff --git a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java index 4a540f2273739..118b4e596fc66 100644 --- a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java @@ -57,6 +57,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.rest.RestStatus; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -188,7 +189,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java b/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java index b01ac39dc515e..0868421fe1d41 100644 --- a/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java +++ b/server/src/test/java/org/opensearch/action/termvectors/TransportMultiTermVectorsActionTests.java @@ -61,6 +61,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -107,7 +108,8 @@ public static void beforeClass() throws Exception { randomBase64UUID() ), null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ) { @Override public TaskManager getTaskManager() { diff --git a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java index a5017867c2e74..4cf82f1dabab3 100644 --- a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java @@ -50,6 +50,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; @@ -552,7 +553,8 @@ private TestTransportService(Transport transport, ThreadPool threadPool) { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID()), null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java index 9e8ef3a325d92..efe91de1ae1a8 100644 --- a/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/opensearch/cluster/action/shard/ShardStateActionTests.java @@ -57,6 +57,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -155,7 +156,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/action/shard/routing/weighted/get/TransportGetWeightedRoutingActionTests.java b/server/src/test/java/org/opensearch/cluster/action/shard/routing/weighted/get/TransportGetWeightedRoutingActionTests.java index 0cde0262a8f38..775d113f986ca 100644 --- a/server/src/test/java/org/opensearch/cluster/action/shard/routing/weighted/get/TransportGetWeightedRoutingActionTests.java +++ b/server/src/test/java/org/opensearch/cluster/action/shard/routing/weighted/get/TransportGetWeightedRoutingActionTests.java @@ -30,6 +30,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; @@ -90,8 +91,8 @@ public void setUpService() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> clusterService.state().nodes().get("nodes1"), null, - Collections.emptySet() - + Collections.emptySet(), + NoopTracer.INSTANCE ); Settings.Builder settingsBuilder = Settings.builder() diff --git a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthTests.java b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthTests.java index 822d9c416d8d0..7910daebb00de 100644 --- a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealthTests.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -64,7 +65,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java index 8afe343ccd56d..b68f0f2375354 100644 --- a/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealthTests.java @@ -29,6 +29,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -82,7 +83,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceDeprecatedMasterTests.java b/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceDeprecatedMasterTests.java index 03b35fe8c9f36..0b84eb19f4264 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceDeprecatedMasterTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceDeprecatedMasterTests.java @@ -36,6 +36,7 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.DiscoveryModule; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; import org.opensearch.transport.TransportRequest; @@ -100,7 +101,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceTests.java b/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceTests.java index be7b32d4aef11..bfb225854979b 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/ClusterBootstrapServiceTests.java @@ -37,6 +37,7 @@ import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.DiscoveryModule; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; import org.opensearch.transport.TransportRequest; @@ -101,7 +102,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java index 30512fd96088e..c152a1606681e 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java @@ -46,6 +46,7 @@ import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.EqualsHashCodeTestUtils; import org.opensearch.test.EqualsHashCodeTestUtils.CopyFunction; import org.opensearch.test.OpenSearchTestCase; @@ -123,7 +124,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -285,7 +287,8 @@ public String toString() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -371,7 +374,8 @@ public String toString() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -475,7 +479,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> follower, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -546,7 +551,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> follower, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -701,7 +707,8 @@ public void testPreferClusterManagerNodes() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> nodes.get(0), null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); final FollowersChecker followersChecker = new FollowersChecker( Settings.EMPTY, diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java index 3fa5768f4f614..78c3b5d45a9ab 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java @@ -50,6 +50,7 @@ import org.opensearch.monitor.StatusInfo; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.test.transport.CapturingTransport.CapturedRequest; @@ -96,7 +97,8 @@ public void testJoinDeduplication() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); JoinHelper joinHelper = new JoinHelper( Settings.EMPTY, @@ -281,7 +283,8 @@ public void testJoinFailureOnUnhealthyNodes() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); AtomicReference nodeHealthServiceStatus = new AtomicReference<>(new StatusInfo(UNHEALTHY, "unhealthy-info")); JoinHelper joinHelper = new JoinHelper( @@ -472,7 +475,8 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); } else { transportService = mockTransport.createTransportService( @@ -481,7 +485,8 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); } JoinHelper joinHelper = new JoinHelper( diff --git a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java index 180d0ffe649e2..8915f4c5c1274 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java @@ -43,6 +43,7 @@ import org.opensearch.core.transport.TransportResponse; import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.monitor.StatusInfo; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.EqualsHashCodeTestUtils; import org.opensearch.test.EqualsHashCodeTestUtils.CopyFunction; import org.opensearch.test.OpenSearchTestCase; @@ -165,7 +166,8 @@ public String toString() { NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -281,7 +283,8 @@ public String toString() { NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -393,7 +396,8 @@ public String toString() { NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); @@ -438,7 +442,8 @@ public void testLeaderBehaviour() { NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java index 766a20fda8d28..d94f3fb304fe2 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java @@ -61,6 +61,7 @@ import org.opensearch.monitor.StatusInfo; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; @@ -247,7 +248,8 @@ protected void onSendRequest( TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> initialState.nodes().getLocalNode(), clusterSettings, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, new InMemoryPersistedState(term, initialState)); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java index 8f779097d50d4..5ddf614db3334 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PreVoteCollectorTests.java @@ -41,6 +41,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.monitor.StatusInfo; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; import org.opensearch.transport.ConnectTransportException; @@ -136,7 +137,8 @@ public void handleRemoteError(long requestId, Throwable t) { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 2ff78d3b68082..6d94054afdea2 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -43,6 +43,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.node.Node; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.transport.TransportService; @@ -68,7 +69,8 @@ public void testDiffSerializationFailure() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, clusterSettings, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); final PublicationTransportHandler handler = new PublicationTransportHandler( transportService, diff --git a/server/src/test/java/org/opensearch/cluster/decommission/DecommissionControllerTests.java b/server/src/test/java/org/opensearch/cluster/decommission/DecommissionControllerTests.java index 7d50ab5dfeb1b..627f31502a417 100644 --- a/server/src/test/java/org/opensearch/cluster/decommission/DecommissionControllerTests.java +++ b/server/src/test/java/org/opensearch/cluster/decommission/DecommissionControllerTests.java @@ -28,6 +28,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.TestThreadPool; @@ -91,7 +92,8 @@ public void setTransportServiceAndDefaultClusterState() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> clusterService.state().nodes().get("node1"), null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); final Settings.Builder nodeSettingsBuilder = Settings.builder(); diff --git a/server/src/test/java/org/opensearch/cluster/decommission/DecommissionServiceTests.java b/server/src/test/java/org/opensearch/cluster/decommission/DecommissionServiceTests.java index 208bbb6a7a96d..6c15d1dc54aea 100644 --- a/server/src/test/java/org/opensearch/cluster/decommission/DecommissionServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/decommission/DecommissionServiceTests.java @@ -28,6 +28,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; @@ -103,7 +104,8 @@ public void setUpService() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> clusterService.state().nodes().get("node1"), null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); final Settings.Builder nodeSettingsBuilder = Settings.builder() diff --git a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java index c1c14188db97c..795dc8a624e38 100644 --- a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java @@ -59,6 +59,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.gateway.TestGatewayAllocator; import org.opensearch.test.transport.CapturingTransport; @@ -117,7 +118,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/routing/WeightedRoutingServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/WeightedRoutingServiceTests.java index b1ba52204c47a..5c0bdc8547f8b 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/WeightedRoutingServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/WeightedRoutingServiceTests.java @@ -31,6 +31,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; @@ -94,8 +95,8 @@ public void setUpService() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> clusterService.state().nodes().get("nodes1"), null, - Collections.emptySet() - + Collections.emptySet(), + NoopTracer.INSTANCE ); Settings.Builder settingsBuilder = Settings.builder() diff --git a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java index 2e3c98db6fa81..d28a4a51999e6 100644 --- a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java +++ b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java @@ -47,6 +47,8 @@ import org.opensearch.http.HttpStats; import org.opensearch.http.NullDispatcher; import org.opensearch.plugins.NetworkPlugin; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -143,7 +145,8 @@ public Map> getHttpTransports( NamedXContentRegistry xContentRegistry, NetworkService networkService, HttpServerTransport.Dispatcher requestDispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { return Collections.singletonMap("custom", custom); } @@ -188,7 +191,8 @@ public Map> getHttpTransports( NamedXContentRegistry xContentRegistry, NetworkService networkService, HttpServerTransport.Dispatcher requestDispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { Map> supplierMap = new HashMap<>(); supplierMap.put("custom", custom); @@ -231,7 +235,8 @@ public Map> getHttpTransports( NamedXContentRegistry xContentRegistry, NetworkService networkService, HttpServerTransport.Dispatcher requestDispatcher, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + Tracer tracer ) { Map> supplierMap = new HashMap<>(); supplierMap.put("custom", custom); @@ -313,7 +318,8 @@ private NetworkModule newNetworkModule(Settings settings, NetworkPlugin... plugi xContentRegistry(), null, new NullDispatcher(), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ); } } diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index 7c0dca3803cb2..b33ebf8333b36 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -48,6 +48,7 @@ import org.opensearch.gateway.GatewayMetaState; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.plugins.DiscoveryPlugin; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.ThreadPool; @@ -96,7 +97,7 @@ default Map> getSeedHostProviders( public void setupDummyServices() { threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); - transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null); + transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, NoopTracer.INSTANCE); clusterManagerService = mock(ClusterManagerService.class); namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList()); clusterApplier = mock(ClusterApplier.class); diff --git a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java index ac2bcfe92ebaf..688a532a61c4a 100644 --- a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java +++ b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -114,7 +115,8 @@ public BoundTransportAddress boundAddress() { transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - null + null, + NoopTracer.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/discovery/HandshakingTransportAddressConnectorTests.java b/server/src/test/java/org/opensearch/discovery/HandshakingTransportAddressConnectorTests.java index cf53bd9251b65..0d694bcfa135b 100644 --- a/server/src/test/java/org/opensearch/discovery/HandshakingTransportAddressConnectorTests.java +++ b/server/src/test/java/org/opensearch/discovery/HandshakingTransportAddressConnectorTests.java @@ -44,6 +44,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; @@ -122,7 +123,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req TransportService.NOOP_TRANSPORT_INTERCEPTOR, address -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); transportService.start(); diff --git a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java index 76520f9684c23..f861ab90896db 100644 --- a/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java +++ b/server/src/test/java/org/opensearch/discovery/PeerFinderTests.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.discovery.PeerFinder.TransportAddressConnector; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.test.transport.CapturingTransport.CapturedRequest; @@ -242,7 +243,8 @@ public void setup() { boundTransportAddress -> localNode, null, emptySet(), - connectionManager + connectionManager, + NoopTracer.INSTANCE ); transportService.start(); diff --git a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java index d2a1be87388ad..dc0829adac101 100644 --- a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java +++ b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java @@ -48,6 +48,7 @@ import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -203,7 +204,8 @@ public BoundTransportAddress boundAddress() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); closeables.push(transportService); final List transportAddresses = SeedHostsResolver.resolveHostsLists( @@ -261,7 +263,8 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); closeables.push(transportService); @@ -326,7 +329,8 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); closeables.push(transportService); final TimeValue resolveTimeout = TimeValue.timeValueSeconds(randomIntBetween(3, 5)); @@ -402,7 +406,8 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); closeables.push(transportService); recreateSeedHostsResolver( @@ -446,7 +451,8 @@ public BoundTransportAddress boundAddress() { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); closeables.push(transportService); final List transportAddresses = SeedHostsResolver.resolveHostsLists( diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 697cc92e82a5e..57883f1e5914a 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -45,6 +45,7 @@ import org.opensearch.identity.IdentityService; import org.opensearch.plugins.ExtensionAwarePlugin; import org.opensearch.rest.RestController; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; @@ -130,7 +131,8 @@ public void setup() throws Exception { Version.CURRENT ), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); actionModule = mock(ActionModule.class); extAwarePlugin = new ExtensionAwarePlugin() { @@ -761,7 +763,8 @@ public void testRegisterHandler() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ) ); extensionsManager.initializeServicesAndRestHandler( diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java index 3a7e994f180d4..1dede94c68208 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java @@ -24,6 +24,7 @@ import org.opensearch.extensions.AcknowledgedResponse; import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.rest.RestSendToExtensionActionTests; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; import org.opensearch.test.transport.MockTransportService; @@ -83,7 +84,8 @@ public void setup() throws Exception { Version.CURRENT ), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); discoveryExtensionNode = new DiscoveryExtensionNode( "firstExtension", diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 455c9eef19bc7..87767978147cd 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -22,6 +22,7 @@ import org.opensearch.extensions.ExtensionsManager; import org.opensearch.extensions.ExtensionsSettings; import org.opensearch.rest.RestRequest; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; import org.opensearch.test.rest.FakeRestRequest; @@ -81,7 +82,8 @@ public void setup() throws Exception { Version.CURRENT ), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index 0e75f0a8dacff..992e37b3f120a 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -32,6 +32,7 @@ import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestHandler.Route; import org.opensearch.rest.RestRequest.Method; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -94,7 +95,8 @@ public void setup() throws Exception { Version.CURRENT ), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); discoveryExtensionNode = new DiscoveryExtensionNode( "firstExtension", diff --git a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java index 7d36795d1a896..c34f13041cb11 100644 --- a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java +++ b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java @@ -52,6 +52,7 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; @@ -173,7 +174,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th threadPool, xContentRegistry(), dispatcher, - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + NoopTracer.INSTANCE ) { @Override @@ -238,7 +240,8 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, channel.sendResponse(emptyResponse(RestStatus.BAD_REQUEST)); } }, - clusterSettings + clusterSettings, + NoopTracer.INSTANCE ) { @Override protected HttpServerChannel bind(InetSocketAddress hostAddress) { diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 99673d4b01690..a1d6be84c9926 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -110,6 +110,7 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.script.ScriptService; import org.opensearch.search.internal.ReaderContext; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; @@ -204,7 +205,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); repositoriesService = new RepositoriesService( settings, diff --git a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java index 73073f0bb9e47..8363ea3757a2b 100644 --- a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java @@ -45,6 +45,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -78,7 +79,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java index 72a0fdaf5c77a..ed04d9a20f18e 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java @@ -49,6 +49,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -85,7 +86,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java index 9895eddb911f5..d9bca55a208c2 100644 --- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java @@ -49,6 +49,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -84,7 +85,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 8de652138e83e..dc4dca80ea110 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -111,6 +111,7 @@ import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.snapshots.EmptySnapshotsInfoService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.gateway.TestGatewayAllocator; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -237,7 +238,8 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(SETTINGS, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); MetadataIndexUpgradeService metadataIndexUpgradeService = new MetadataIndexUpgradeService( SETTINGS, diff --git a/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index ee6595ade1d34..22bf337b05598 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -70,6 +70,7 @@ import org.opensearch.indices.replication.SegmentReplicationTargetService; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -545,7 +546,8 @@ private IndicesClusterStateService createIndicesClusterStateService( TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); final ClusterService clusterService = mock(ClusterService.class); final RepositoriesService repositoriesService = new RepositoriesService( diff --git a/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java index 67729bbd5f76a..bcacef83d190a 100644 --- a/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java @@ -23,6 +23,7 @@ import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.transport.TransportService; @@ -68,7 +69,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java index f2e0baf28ecd1..8f84053f2618e 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceServiceTests.java @@ -29,6 +29,7 @@ import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.CopyStateTests; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; @@ -102,7 +103,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 6314feeed3b09..c1f88a6938d33 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -37,6 +37,7 @@ import org.opensearch.indices.replication.common.ReplicationFailedException; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -117,7 +118,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java index 401700c2b2ab9..2cf006176022d 100644 --- a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java @@ -25,6 +25,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationTargetService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.threadpool.TestThreadPool; @@ -61,7 +62,8 @@ public void setUp() throws Exception { TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index d14858a4441b5..889d0dc6ddb14 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -77,6 +77,7 @@ import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -124,7 +125,8 @@ private RepositoriesService createRepositoriesServiceWithMockedClusterService(Cl TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); final ClusterApplierService clusterApplierService = mock(ClusterApplierService.class); when(clusterApplierService.threadPool()).thenReturn(threadPool); diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java index 2d125b8d36542..f0d930c4c3acb 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java @@ -32,6 +32,7 @@ import org.opensearch.tasks.TaskCancellationService; import org.opensearch.tasks.TaskManager; import org.opensearch.tasks.TaskResourceTrackingService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -70,7 +71,7 @@ public class SearchBackpressureServiceTests extends OpenSearchTestCase { @Before public void setup() { threadPool = new TestThreadPool(getClass().getName()); - transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool); + transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, NoopTracer.INSTANCE); transportService.start(); transportService.acceptIncomingRequests(); taskManager = transportService.getTaskManager(); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 9cc7a83d3c563..3c31c979ce856 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -221,6 +221,7 @@ import org.opensearch.search.query.QueryPhase; import org.opensearch.snapshots.mockstore.MockEventuallyConsistentRepository; import org.opensearch.tasks.TaskResourceTrackingService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.disruption.DisruptableMockTransport; import org.opensearch.threadpool.ThreadPool; @@ -1985,7 +1986,7 @@ public void onFailure(final Exception e) { return actualHandler; } } - }, a -> node, null, emptySet()); + }, a -> node, null, emptySet(), NoopTracer.INSTANCE); final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver( new ThreadContext(Settings.EMPTY) ); diff --git a/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java b/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java index aa626013240b8..bb154b95f9f01 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java @@ -14,6 +14,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.tasks.TaskId; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.Scheduler; @@ -47,7 +48,7 @@ public class TaskCancellationMonitoringServiceTests extends OpenSearchTestCase { @Before public void setup() { threadPool = new TestThreadPool(getClass().getName()); - transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool); + transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, NoopTracer.INSTANCE); transportService.start(); transportService.acceptIncomingRequests(); taskManager = transportService.getTaskManager(); diff --git a/server/src/test/java/org/opensearch/transport/ProxyConnectionStrategyTests.java b/server/src/test/java/org/opensearch/transport/ProxyConnectionStrategyTests.java index 510a2b3abd943..1c9880ed14714 100644 --- a/server/src/test/java/org/opensearch/transport/ProxyConnectionStrategyTests.java +++ b/server/src/test/java/org/opensearch/transport/ProxyConnectionStrategyTests.java @@ -42,6 +42,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -82,7 +83,7 @@ public MockTransportService startTransport(final String id, final Version versio .put("node.name", id) .put(settings) .build(); - MockTransportService newService = MockTransportService.createNewService(s, version, threadPool); + MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, NoopTracer.INSTANCE); try { newService.start(); newService.acceptIncomingRequests(); @@ -99,7 +100,14 @@ public void testProxyStrategyWillOpenExpectedNumberOfConnectionsToAddress() { try (MockTransportService transport1 = startTransport("node1", Version.CURRENT)) { TransportAddress address1 = transport1.boundAddress().publishAddress(); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -138,7 +146,14 @@ public void testProxyStrategyWillOpenNewConnectionsOnDisconnect() throws Excepti TransportAddress address1 = transport1.boundAddress().publishAddress(); TransportAddress address2 = transport2.boundAddress().publishAddress(); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -200,7 +215,14 @@ public void testConnectFailsWithIncompatibleNodes() { try (MockTransportService transport1 = startTransport("incompatible-node", incompatibleVersion)) { TransportAddress address1 = transport1.boundAddress().publishAddress(); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -240,7 +262,14 @@ public void testClusterNameValidationPreventConnectingToDifferentClusters() thro TransportAddress address1 = transport1.boundAddress().publishAddress(); TransportAddress address2 = transport2.boundAddress().publishAddress(); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -303,7 +332,14 @@ public void testProxyStrategyWillResolveAddressesEachConnect() throws Exception return address; }; - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -338,7 +374,14 @@ public void testProxyStrategyWillNeedToBeRebuiltIfNumOfSocketsOrAddressesOrServe try (MockTransportService remoteTransport = startTransport("node1", Version.CURRENT)) { TransportAddress remoteAddress = remoteTransport.boundAddress().publishAddress(); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -441,7 +484,14 @@ public void testServerNameAttributes() { try (MockTransportService transport1 = startTransport("node1", Version.CURRENT, bindSettings)) { TransportAddress address1 = transport1.boundAddress().publishAddress(); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/transport/RemoteClusterAwareClientTests.java b/server/src/test/java/org/opensearch/transport/RemoteClusterAwareClientTests.java index a2a77168c8991..7595982837365 100644 --- a/server/src/test/java/org/opensearch/transport/RemoteClusterAwareClientTests.java +++ b/server/src/test/java/org/opensearch/transport/RemoteClusterAwareClientTests.java @@ -41,6 +41,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -81,7 +82,14 @@ public void testSearchShards() throws Exception { Collections.shuffle(knownNodes, random()); Settings.Builder builder = Settings.builder(); builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString()); - try (MockTransportService service = MockTransportService.createNewService(builder.build(), Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + builder.build(), + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); @@ -121,7 +129,14 @@ public void testSearchShardsThreadContextHeader() { Collections.shuffle(knownNodes, random()); Settings.Builder builder = Settings.builder(); builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString()); - try (MockTransportService service = MockTransportService.createNewService(builder.build(), Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + builder.build(), + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/opensearch/transport/RemoteClusterClientTests.java index b89d652510850..f3b7f9916d460 100644 --- a/server/src/test/java/org/opensearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/opensearch/transport/RemoteClusterClientTests.java @@ -39,6 +39,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.settings.Settings; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; import org.opensearch.test.transport.MockTransportService; @@ -79,7 +80,14 @@ public void testConnectAndExecuteRequest() throws Exception { .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)) .put("cluster.remote.test.seeds", remoteNode.getAddress().getAddress() + ":" + remoteNode.getAddress().getPort()) .build(); - try (MockTransportService service = MockTransportService.createNewService(localSettings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + localSettings, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); // following two log lines added to investigate #41745, can be removed once issue is closed logger.info("Start accepting incoming requests on local transport service"); @@ -118,7 +126,14 @@ public void testEnsureWeReconnect() throws Exception { .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)) .put("cluster.remote.test.seeds", remoteNode.getAddress().getAddress() + ":" + remoteNode.getAddress().getPort()) .build(); - try (MockTransportService service = MockTransportService.createNewService(localSettings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + localSettings, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); // this test is not perfect since we might reconnect concurrently but it will fail most of the time if we don't have // the right calls in place in the RemoteAwareClient @@ -147,7 +162,9 @@ public void testEnsureWeReconnect() throws Exception { public void testRemoteClusterServiceNotEnabled() { final Settings settings = removeRoles(Collections.singleton(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)); - try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, NoopTracer.INSTANCE) + ) { service.start(); service.acceptIncomingRequests(); final RemoteClusterService remoteClusterService = service.getRemoteClusterService(); diff --git a/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java index d481f361f2e54..bb653439ec21e 100644 --- a/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/opensearch/transport/RemoteClusterConnectionTests.java @@ -66,6 +66,7 @@ import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.internal.InternalSearchResponse; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -126,7 +127,7 @@ public static MockTransportService startTransport( boolean success = false; final Settings s = Settings.builder().put(settings).put("node.name", id).build(); ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s); - MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, null); + MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, NoopTracer.INSTANCE); try { newService.registerRequestHandler( ClusterSearchShardsAction.NAME, @@ -231,7 +232,14 @@ public void run() { }; t.start(); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); CountDownLatch listenerCalled = new CountDownLatch(1); @@ -280,7 +288,14 @@ public void testCloseWhileConcurrentlyConnecting() throws IOException, Interrupt List seedNodes = addresses(seedNode1, seedNode); Collections.shuffle(seedNodes, random()); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); String clusterAlias = "test-cluster"; @@ -367,7 +382,14 @@ public void testGetConnectionInfo() throws Exception { List seedNodes = addresses(node3, node1, node2); Collections.shuffle(seedNodes, random()); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); int maxNumConnections = randomIntBetween(1, 5); @@ -480,7 +502,14 @@ public void testCollectNodes() throws Exception { try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT)) { DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); knownNodes.add(seedTransport.getLocalDiscoNode()); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); String clusterAlias = "test-cluster"; @@ -515,7 +544,14 @@ public void testNoChannelsExceptREG() throws Exception { try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT)) { DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); knownNodes.add(seedTransport.getLocalDiscoNode()); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); String clusterAlias = "test-cluster"; @@ -568,7 +604,14 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted ); Collections.shuffle(seedNodes, random()); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); @@ -645,7 +688,14 @@ public void testGetConnection() throws Exception { DiscoveryNode disconnectedNode = disconnectedTransport.getLocalNode(); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); String clusterAlias = "test-cluster"; diff --git a/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java index f6f3e8fa60863..449715189c881 100644 --- a/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/opensearch/transport/RemoteClusterServiceTests.java @@ -44,6 +44,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -162,7 +163,7 @@ public void testGroupClusterIndices() throws IOException { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -233,7 +234,7 @@ public void testGroupIndices() throws IOException { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -326,7 +327,7 @@ public void testIncrementallyAddClusters() throws IOException { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -393,7 +394,12 @@ public void testDefaultPingSchedule() throws IOException { } Settings settings = settingsBuilder.build(); try ( - MockTransportService transportService = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null) + MockTransportService transportService = MockTransportService.createNewService( + settings, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) ) { transportService.start(); transportService.acceptIncomingRequests(); @@ -436,7 +442,7 @@ public void testCustomPingSchedule() throws IOException { transportSettings, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -474,7 +480,7 @@ public void testChangeSettings() throws Exception { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -523,7 +529,12 @@ public void testRemoteNodeAttribute() throws IOException, InterruptedException { Collections.shuffle(knownNodes, random()); try ( - MockTransportService transportService = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null) + MockTransportService transportService = MockTransportService.createNewService( + settings, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) ) { transportService.start(); transportService.acceptIncomingRequests(); @@ -586,7 +597,12 @@ public void testRemoteNodeRoles() throws IOException, InterruptedException { Collections.shuffle(knownNodes, random()); try ( - MockTransportService transportService = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null) + MockTransportService transportService = MockTransportService.createNewService( + settings, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) ) { transportService.start(); transportService.acceptIncomingRequests(); @@ -654,7 +670,12 @@ public void testCollectNodes() throws InterruptedException, IOException { Collections.shuffle(knownNodes_c2, random()); try ( - MockTransportService transportService = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null) + MockTransportService transportService = MockTransportService.createNewService( + settings, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) ) { transportService.start(); transportService.acceptIncomingRequests(); @@ -901,7 +922,7 @@ public void testReconnectWhenStrategySettingsUpdated() throws Exception { Settings.EMPTY, Version.CURRENT, threadPool, - null + NoopTracer.INSTANCE ) ) { transportService.start(); @@ -983,7 +1004,14 @@ public void testSkipUnavailable() { knownNodes.add(seedNode); Settings.Builder builder = Settings.builder(); builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString()); - try (MockTransportService service = MockTransportService.createNewService(builder.build(), Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService( + builder.build(), + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { service.start(); service.acceptIncomingRequests(); @@ -1002,7 +1030,9 @@ public void testSkipUnavailable() { public void testRemoteClusterServiceNotEnabledGetRemoteClusterConnection() { final Settings settings = removeRoles(Collections.singleton(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)); - try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, NoopTracer.INSTANCE) + ) { service.start(); service.acceptIncomingRequests(); final IllegalArgumentException e = expectThrows( @@ -1015,7 +1045,9 @@ public void testRemoteClusterServiceNotEnabledGetRemoteClusterConnection() { public void testRemoteClusterServiceNotEnabledGetCollectNodes() { final Settings settings = removeRoles(Collections.singleton(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)); - try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + try ( + MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, NoopTracer.INSTANCE) + ) { service.start(); service.acceptIncomingRequests(); final IllegalArgumentException e = expectThrows( diff --git a/server/src/test/java/org/opensearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/opensearch/transport/SniffConnectionStrategyTests.java index ca85ec2270caf..c89a9d328b419 100644 --- a/server/src/test/java/org/opensearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/opensearch/transport/SniffConnectionStrategyTests.java @@ -49,6 +49,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.opensearch.test.transport.MockTransportService; @@ -105,7 +106,7 @@ public MockTransportService startTransport( .put(settings) .build(); ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s); - MockTransportService newService = MockTransportService.createNewService(s, version, threadPool); + MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, NoopTracer.INSTANCE); try { newService.registerRequestHandler( ClusterStateAction.NAME, @@ -143,7 +144,14 @@ public void testSniffStrategyWillConnectToAndDiscoverNodes() { knownNodes.add(discoverableNode); Collections.shuffle(knownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -192,7 +200,14 @@ public void testSniffStrategyWillResolveDiscoveryNodesEachConnect() throws Excep return seedNode; }; - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -240,7 +255,14 @@ public void testSniffStrategyWillConnectToMaxAllowedNodesAndOpenNewConnectionsOn knownNodes.add(discoverableNode2); Collections.shuffle(knownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -297,7 +319,14 @@ public void testDiscoverWithSingleIncompatibleSeedNode() { knownNodes.add(discoverableNode); Collections.shuffle(knownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -336,7 +365,14 @@ public void testConnectFailsWithIncompatibleNodes() { DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalNode(); knownNodes.add(incompatibleSeedNode); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -378,7 +414,14 @@ public void testFilterNodesWithNodePredicate() { DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode; Collections.shuffle(knownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -424,7 +467,14 @@ public void testConnectFailsIfNoConnectionsOpened() { knownNodes.add(discoverableNode); closedTransport.close(); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -474,7 +524,14 @@ public void testClusterNameValidationPreventConnectingToDifferentClusters() thro Collections.shuffle(knownNodes, random()); Collections.shuffle(otherKnownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -542,7 +599,14 @@ public void testMultipleCallsToConnectEnsuresConnection() { knownNodes.add(discoverableNode); Collections.shuffle(knownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -589,8 +653,18 @@ public void testConfiguredProxyAddressModeWillReplaceNodeAddress() { List knownNodes = new CopyOnWriteArrayList<>(); try ( MockTransportService accessible = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService unresponsive1 = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool); - MockTransportService unresponsive2 = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool) + MockTransportService unresponsive1 = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ); + MockTransportService unresponsive2 = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) ) { // We start in order to get a valid address + port, but do not start accepting connections as we // will not actually connect to these transports @@ -616,7 +690,14 @@ public void testConfiguredProxyAddressModeWillReplaceNodeAddress() { knownNodes.add(discoverableNode); Collections.shuffle(knownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); @@ -679,7 +760,14 @@ public void testSniffStrategyWillNeedToBeRebuiltIfNumOfConnectionsOrSeedsOrProxy knownNodes.add(discoverableNode); Collections.shuffle(knownNodes, random()); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + try ( + MockTransportService localService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { localService.start(); localService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java index c23c284be1ca5..dd2aefd2318f7 100644 --- a/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportActionProxyTests.java @@ -40,6 +40,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; @@ -85,7 +86,7 @@ public void tearDown() throws Exception { } private MockTransportService buildService(final Version version) { - MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, version, threadPool, null); + MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, version, threadPool, NoopTracer.INSTANCE); service.start(); service.acceptIncomingRequests(); return service; diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java index c0dd854139ee5..d10b4f26100cc 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceDeserializationFailureTests.java @@ -43,6 +43,7 @@ import org.opensearch.core.transport.TransportResponse; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskAwareRequest; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.ThreadPool; @@ -82,7 +83,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req TransportService.NOOP_TRANSPORT_INTERCEPTOR, ignored -> localNode, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.registerRequestHandler( diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java index 5abb032120dcf..67db155779ec6 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java @@ -43,6 +43,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -100,7 +101,8 @@ private NetworkHandle startServices(String nodeNameAndId, Settings settings, Ver version ), null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 1f56615959618..d24cc24d28579 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -88,6 +88,7 @@ import org.opensearch.monitor.StatusInfo; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.disruption.DisruptableMockTransport; import org.opensearch.test.disruption.DisruptableMockTransport.ConnectionStatus; @@ -1116,7 +1117,8 @@ protected Optional getDisruptableMockTransport(Transpo getTransportInterceptor(localNode, threadPool), a -> localNode, null, - emptySet() + emptySet(), + NoopTracer.INSTANCE ); clusterManagerService = new AckedFakeThreadPoolClusterManagerService( localNode.getId(), diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java index 803a613ba55ff..e6c7e21d5b3ea 100644 --- a/test/framework/src/main/java/org/opensearch/node/MockNode.java +++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java @@ -60,6 +60,7 @@ import org.opensearch.search.SearchService; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.query.QueryPhase; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.test.MockHttpTransport; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.ThreadPool; @@ -199,16 +200,35 @@ protected TransportService newTransportService( TransportInterceptor interceptor, Function localNodeFactory, ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { // we use the MockTransportService.TestPlugin class as a marker to create a network // module with this MockNetworkService. NetworkService is such an integral part of the systme // we don't allow to plug it in from plugins or anything. this is a test-only override and // can't be done in a production env. if (getPluginsService().filterPlugins(MockTransportService.TestPlugin.class).isEmpty()) { - return super.newTransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); + return super.newTransportService( + settings, + transport, + threadPool, + interceptor, + localNodeFactory, + clusterSettings, + taskHeaders, + tracer + ); } else { - return new MockTransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); + return new MockTransportService( + settings, + transport, + threadPool, + interceptor, + localNodeFactory, + clusterSettings, + taskHeaders, + tracer + ); } } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 65e5a18c89949..25f453fe024ff 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2090,7 +2090,11 @@ protected boolean addMockGeoShapeFieldMapper() { return true; } - /** Returns {@code true} if this test cluster should have tracing enabled with MockTelemetryPlugin */ + /** + * Returns {@code true} if this test cluster should have tracing enabled with MockTelemetryPlugin + * Disabling this for now as the existing way of strict check do not support multiple nodes internal cluster. + * @return boolean. + */ protected boolean addMockTelemetryPlugin() { return true; } diff --git a/test/framework/src/main/java/org/opensearch/test/disruption/DisruptableMockTransport.java b/test/framework/src/main/java/org/opensearch/test/disruption/DisruptableMockTransport.java index 2029d9893ea35..4f3884f97a570 100644 --- a/test/framework/src/main/java/org/opensearch/test/disruption/DisruptableMockTransport.java +++ b/test/framework/src/main/java/org/opensearch/test/disruption/DisruptableMockTransport.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.test.transport.MockTransport; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.CloseableConnection; @@ -91,9 +92,10 @@ public TransportService createTransportService( TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { - return new TransportService(settings, this, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); + return new TransportService(settings, this, threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, tracer); } @Override diff --git a/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java b/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java index 42a7b63a3d762..24aef714cc259 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/MockTransport.java @@ -45,6 +45,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.BoundTransportAddress; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.CloseableConnection; import org.opensearch.transport.ClusterConnectionManager; @@ -80,7 +81,8 @@ public TransportService createTransportService( TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { StubbableConnectionManager connectionManager = new StubbableConnectionManager(new ClusterConnectionManager(settings, this)); connectionManager.setDefaultNodeConnectedBehavior((cm, node) -> false); @@ -93,7 +95,8 @@ public TransportService createTransportService( localNodeFactory, clusterSettings, taskHeaders, - connectionManager + connectionManager, + tracer ); } diff --git a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java index 0b4d801fb75f1..c1795e61096ac 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java @@ -57,6 +57,7 @@ import org.opensearch.node.Node; import org.opensearch.plugins.Plugin; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.tasks.MockTaskManager; import org.opensearch.threadpool.ThreadPool; @@ -114,18 +115,19 @@ public List> getSettings() { } } - public static MockTransportService createNewService(Settings settings, Version version, ThreadPool threadPool) { - return createNewService(settings, version, threadPool, null); + public static MockTransportService createNewService(Settings settings, Version version, ThreadPool threadPool, Tracer tracer) { + return createNewService(settings, version, threadPool, null, tracer); } public static MockTransportService createNewService( Settings settings, Version version, ThreadPool threadPool, - @Nullable ClusterSettings clusterSettings + @Nullable ClusterSettings clusterSettings, + Tracer tracer ) { MockNioTransport mockTransport = newMockTransport(settings, version, threadPool); - return createNewService(settings, mockTransport, version, threadPool, clusterSettings, Collections.emptySet()); + return createNewService(settings, mockTransport, version, threadPool, clusterSettings, Collections.emptySet(), tracer); } public static MockNioTransport newMockTransport(Settings settings, Version version, ThreadPool threadPool) { @@ -148,9 +150,10 @@ public static MockTransportService createNewService( Version version, ThreadPool threadPool, @Nullable ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { - return createNewService(settings, transport, version, threadPool, clusterSettings, taskHeaders, NOOP_TRANSPORT_INTERCEPTOR); + return createNewService(settings, transport, version, threadPool, clusterSettings, taskHeaders, NOOP_TRANSPORT_INTERCEPTOR, tracer); } public static MockTransportService createNewService( @@ -160,7 +163,8 @@ public static MockTransportService createNewService( ThreadPool threadPool, @Nullable ClusterSettings clusterSettings, Set taskHeaders, - TransportInterceptor interceptor + TransportInterceptor interceptor, + Tracer tracer ) { return new MockTransportService( settings, @@ -176,7 +180,8 @@ public static MockTransportService createNewService( version ), clusterSettings, - taskHeaders + taskHeaders, + tracer ); } @@ -194,7 +199,8 @@ public MockTransportService( Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, - @Nullable ClusterSettings clusterSettings + @Nullable ClusterSettings clusterSettings, + Tracer tracer ) { this( settings, @@ -207,7 +213,8 @@ public MockTransportService( settings.get(Node.NODE_NAME_SETTING.getKey(), UUIDs.randomBase64UUID()) ), clusterSettings, - Collections.emptySet() + Collections.emptySet(), + tracer ); } @@ -225,9 +232,10 @@ public MockTransportService( TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { - this(settings, new StubbableTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders); + this(settings, new StubbableTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings, taskHeaders, tracer); } private MockTransportService( @@ -237,7 +245,8 @@ private MockTransportService( TransportInterceptor interceptor, Function localNodeFactory, @Nullable ClusterSettings clusterSettings, - Set taskHeaders + Set taskHeaders, + Tracer tracer ) { super( settings, @@ -247,7 +256,8 @@ private MockTransportService( localNodeFactory, clusterSettings, taskHeaders, - new StubbableConnectionManager(new ClusterConnectionManager(settings, transport)) + new StubbableConnectionManager(new ClusterConnectionManager(settings, transport)), + tracer ); this.original = transport.getDelegate(); } diff --git a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java index 3bc8901e6a29b..3b64e044e7bf0 100644 --- a/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/opensearch/transport/AbstractSimpleTransportTestCase.java @@ -67,6 +67,7 @@ import org.opensearch.core.transport.TransportResponse; import org.opensearch.node.Node; import org.opensearch.tasks.Task; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -227,7 +228,8 @@ private MockTransportService buildService( threadPool, clusterSettings, Collections.emptySet(), - interceptor + interceptor, + NoopTracer.INSTANCE ); service.start(); if (acceptRequests) { diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java index 516cfe1636bf7..6b64270ca68e1 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/DisruptableMockTransportTests.java @@ -43,6 +43,7 @@ import org.opensearch.core.transport.TransportResponse; import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.node.Node; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.disruption.DisruptableMockTransport.ConnectionStatus; import org.opensearch.threadpool.ThreadPool; @@ -161,7 +162,8 @@ protected void execute(Runnable runnable) { NOOP_TRANSPORT_INTERCEPTOR, a -> node1, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); service2 = transport2.createTransportService( Settings.EMPTY, @@ -169,7 +171,8 @@ protected void execute(Runnable runnable) { NOOP_TRANSPORT_INTERCEPTOR, a -> node2, null, - Collections.emptySet() + Collections.emptySet(), + NoopTracer.INSTANCE ); service1.start(); diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java index 892c1a8b3eeae..694a6a2b9e45b 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java @@ -194,4 +194,12 @@ private static String generateTraceId() { public Object getAttribute(String key) { return metadata.get(key); } + + /** + * Returns the attributes as map. + * @return returns the attributes map. + */ + public Map getAttributes() { + return metadata; + } } diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpanData.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpanData.java index bc71d097ac28b..0658a6421f3f3 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpanData.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpanData.java @@ -9,6 +9,7 @@ package org.opensearch.test.telemetry.tracing; import java.util.Arrays; +import java.util.Map; /** * MockSpanData model for storing Telemetry information for testing. @@ -17,6 +18,7 @@ public class MockSpanData { /** * MockSpanData constructor with spanID, parentSpanID, traceID, startEpochNanos, endEpochNanos, hasEnded params. + * * @param spanID spanID * @param parentSpanID spanID of the parentSpan * @param traceID traceID of the request @@ -24,6 +26,7 @@ public class MockSpanData { * @param endEpochNanos endTime of span in epochNanos * @param hasEnded value if the span is closed * @param spanName Name of the span emitted + * @param attributes span attributes */ public MockSpanData( String spanID, @@ -32,7 +35,8 @@ public MockSpanData( long startEpochNanos, long endEpochNanos, boolean hasEnded, - String spanName + String spanName, + Map attributes ) { this.spanID = spanID; this.traceID = traceID; @@ -41,10 +45,12 @@ public MockSpanData( this.endEpochNanos = endEpochNanos; this.hasEnded = hasEnded; this.spanName = spanName; + this.attributes = attributes; } /** * MockSpanData constructor with spanID, parentSpanID, traceID, startEpochNanos, hasEnded and spanName params. + * * @param spanID spanID * @param parentSpanID spanID of the parentSpan * @param traceID traceID of the request @@ -52,6 +58,7 @@ public MockSpanData( * @param hasEnded value if the span is closed * @param spanName Name of the span emitted * @param stackTrace StackTrace to debug the problematic span + * @param attributes span attributes */ public MockSpanData( String spanID, @@ -60,7 +67,8 @@ public MockSpanData( long startEpochNanos, boolean hasEnded, String spanName, - StackTraceElement[] stackTrace + StackTraceElement[] stackTrace, + Map attributes ) { this.spanID = spanID; this.traceID = traceID; @@ -69,6 +77,7 @@ public MockSpanData( this.hasEnded = hasEnded; this.spanName = spanName; this.stackTrace = stackTrace; + this.attributes = attributes; } private final String spanID; @@ -79,6 +88,7 @@ public MockSpanData( private final long startEpochNanos; private long endEpochNanos; private boolean hasEnded; + private Map attributes; private StackTraceElement[] stackTrace; @@ -147,6 +157,14 @@ public void setHasEnded(boolean hasEnded) { this.hasEnded = hasEnded; } + /** + * Returns the attributes + * @return returns the attributes map. + */ + public Map getAttributes() { + return attributes; + } + @Override public String toString() { return "MockSpanData{" @@ -168,6 +186,8 @@ public String toString() { + endEpochNanos + ", hasEnded=" + hasEnded + + ", attributes=" + + attributes + ", stackTrace=" + Arrays.toString(stackTrace) + '}'; diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java index c6e57531d23df..f7ebb3ee18a9b 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java @@ -60,7 +60,8 @@ private MockSpanData toMockSpanData(Span span) { System.nanoTime(), false, span.getSpanName(), - Thread.currentThread().getStackTrace() + Thread.currentThread().getStackTrace(), + (span instanceof MockSpan) ? ((MockSpan) span).getAttributes() : Map.of() ); return spanData; } diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/validators/NumberOfTraceIDsEqualToRequests.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/validators/NumberOfTraceIDsEqualToRequests.java index 3e18e4b873557..5fe268a8f0581 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/validators/NumberOfTraceIDsEqualToRequests.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/validators/NumberOfTraceIDsEqualToRequests.java @@ -8,11 +8,13 @@ package org.opensearch.test.telemetry.tracing.validators; +import org.opensearch.telemetry.tracing.attributes.Attributes; import org.opensearch.test.telemetry.tracing.MockSpanData; import org.opensearch.test.telemetry.tracing.TracingValidator; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -21,10 +23,16 @@ */ public class NumberOfTraceIDsEqualToRequests implements TracingValidator { + private static final String FILTERING_ATTRIBUTE = "action"; + private final Attributes attributes; + /** - * Base Constructor + * Constructor. + * @param attributes attributes. */ - public NumberOfTraceIDsEqualToRequests() {} + public NumberOfTraceIDsEqualToRequests(Attributes attributes) { + this.attributes = attributes; + } /** * validates if all spans emitted for a particular request have same traceID. @@ -33,11 +41,25 @@ public NumberOfTraceIDsEqualToRequests() {} */ @Override public List validate(List spans, int requests) { - Set totalTraceIDs = spans.stream().map(MockSpanData::getTraceID).collect(Collectors.toSet()); + Set totalTraceIDs = spans.stream() + .filter(span -> isMatchingSpan(span)) + .map(MockSpanData::getTraceID) + .collect(Collectors.toSet()); List problematicSpans = new ArrayList<>(); if (totalTraceIDs.size() != requests) { problematicSpans.addAll(spans); } return problematicSpans; } + + private boolean isMatchingSpan(MockSpanData mockSpanData) { + if (attributes.getAttributesMap().isEmpty()) { + return true; + } else { + return Objects.equals( + mockSpanData.getAttributes().get(FILTERING_ATTRIBUTE), + attributes.getAttributesMap().get(FILTERING_ATTRIBUTE) + ); + } + } } From 5ad472f0bae642d277211cbed8cc2178449c0f46 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Fri, 8 Sep 2023 13:58:44 -0700 Subject: [PATCH 005/388] Remove custom-codecs plugin (#9944) This plugin has been moved to its own [repository][1]. [1]: https://github.com/opensearch-project/custom-codecs Signed-off-by: Andrew Ross --- CHANGELOG.md | 2 +- modules/reindex/build.gradle | 2 - .../index/codec/MultiCodecReindexIT.java | 7 +- plugins/custom-codecs/build.gradle | 27 -- .../index/codec/CodecCompressionLevelIT.java | 188 ------------- .../index/codec/MultiCodecMergeIT.java | 188 ------------- .../codec/customcodecs/CustomCodecPlugin.java | 47 ---- .../customcodecs/CustomCodecService.java | 72 ----- .../CustomCodecServiceFactory.java | 27 -- .../customcodecs/Lucene95CustomCodec.java | 121 --------- .../Lucene95CustomStoredFieldsFormat.java | 135 ---------- .../index/codec/customcodecs/ZstdCodec.java | 67 ----- .../customcodecs/ZstdCompressionMode.java | 210 --------------- .../customcodecs/ZstdDeprecatedCodec.java | 61 ----- .../codec/customcodecs/ZstdNoDictCodec.java | 67 ----- .../ZstdNoDictCompressionMode.java | 182 ------------- .../codec/customcodecs/package-info.java | 12 - .../plugin-metadata/plugin-security.policy | 11 - .../services/org.apache.lucene.codecs.Codec | 3 - .../customcodecs/AbstractCompressorTests.java | 219 --------------- .../codec/customcodecs/CustomCodecTests.java | 250 ------------------ ...Lucene95CustomStoredFieldsFormatTests.java | 47 ---- .../customcodecs/ZstdCompressorTests.java | 30 --- .../ZstdNoDictCompressorTests.java | 30 --- 24 files changed, 2 insertions(+), 2003 deletions(-) delete mode 100644 plugins/custom-codecs/build.gradle delete mode 100644 plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java delete mode 100644 plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java delete mode 100644 plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy delete mode 100644 plugins/custom-codecs/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec delete mode 100644 plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java delete mode 100644 plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java delete mode 100644 plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java delete mode 100644 plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java delete mode 100644 plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5abbeee490a3d..2404c57ee348d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -186,7 +186,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Store] Add support for Remote Translog Store stats in `_remotestore/stats/` API ([#9263](https://github.com/opensearch-project/OpenSearch/pull/9263)) - Add support for query profiler with concurrent aggregation ([#9248](https://github.com/opensearch-project/OpenSearch/pull/9248)) - Cleanup Unreferenced file on segment merge failure ([#9503](https://github.com/opensearch-project/OpenSearch/pull/9503)) -- Move ZStd to a plugin ([#9658](https://github.com/opensearch-project/OpenSearch/pull/9658)) +- Move zstd compression codec to external custom-codecs repository ([#9422](https://github.com/opensearch-project/OpenSearch/issues/9422]) - [Remote Store] Add support for Remote Translog Store upload stats in `_nodes/stats/` API ([#8908](https://github.com/opensearch-project/OpenSearch/pull/8908)) - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) - Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916)) diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index ea517ca53c000..cad7d67f3ef84 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -69,7 +69,6 @@ dependencies { testImplementation project(':modules:transport-netty4') // for parent/child testing testImplementation project(':modules:parent-join') - testImplementation project(':plugins:custom-codecs') } restResources { @@ -96,5 +95,4 @@ forbiddenPatterns { tasks.named("bundlePlugin").configure { dependsOn("copyParentJoinMetadata") dependsOn("copyTransportNetty4Metadata") - dependsOn("copyCustomCodecsMetadata") } diff --git a/modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java index 3f5df7cf57897..604c233ca49c4 100644 --- a/modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecReindexIT.java @@ -15,7 +15,6 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.customcodecs.CustomCodecPlugin; import org.opensearch.index.engine.Segment; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.reindex.ReindexAction; @@ -47,7 +46,7 @@ public class MultiCodecReindexIT extends ReindexTestCase { @Override protected Collection> nodePlugins() { - return List.of(CustomCodecPlugin.class, ReindexModulePlugin.class); + return List.of(ReindexModulePlugin.class); } public void testReindexingMultipleCodecs() throws InterruptedException, ExecutionException { @@ -57,10 +56,6 @@ public void testReindexingMultipleCodecs() throws InterruptedException, Executio "BEST_COMPRESSION", "zlib", "BEST_COMPRESSION", - "zstd_no_dict", - "ZSTD_NO_DICT", - "zstd", - "ZSTD", "default", "BEST_SPEED", "lz4", diff --git a/plugins/custom-codecs/build.gradle b/plugins/custom-codecs/build.gradle deleted file mode 100644 index 253822e88b817..0000000000000 --- a/plugins/custom-codecs/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.internal-cluster-test' - -opensearchplugin { - name 'custom-codecs' - description 'A plugin that implements custom compression codecs.' - classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin' - licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') -} - -dependencies { - api "com.github.luben:zstd-jni:1.5.5-5" -} - -testingConventions.enabled = false; diff --git a/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java b/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java deleted file mode 100644 index 7810b5810fffb..0000000000000 --- a/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec; - -import org.apache.logging.log4j.core.util.Throwables; -import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.customcodecs.CustomCodecPlugin; -import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.Collection; -import java.util.Collections; -import java.util.concurrent.ExecutionException; - -import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; -import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; - -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) -public class CodecCompressionLevelIT extends OpenSearchIntegTestCase { - @Override - protected Collection> nodePlugins() { - return Collections.singletonList(CustomCodecPlugin.class); - } - - public void testLuceneCodecsCreateIndexWithCompressionLevel() { - - internalCluster().ensureAtLeastNumDataNodes(1); - final String index = "test-index"; - - // creating index - assertThrows( - IllegalArgumentException.class, - () -> createIndex( - index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) - .put("index.codec.compression_level", randomIntBetween(1, 6)) - .build() - ) - ); - - createIndex( - index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) - .build() - ); - ensureGreen(index); - } - - public void testZStandardCodecsCreateIndexWithCompressionLevel() { - - internalCluster().ensureAtLeastNumDataNodes(1); - final String index = "test-index"; - - // creating index - createIndex( - index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(ZSTD_CODEC, ZSTD_NO_DICT_CODEC)) - .put("index.codec.compression_level", randomIntBetween(1, 6)) - .build() - ); - - ensureGreen(index); - } - - public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionException, InterruptedException { - - internalCluster().ensureAtLeastNumDataNodes(1); - final String index = "test-index"; - - // creating index - createIndex( - index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(ZSTD_CODEC, ZSTD_NO_DICT_CODEC)) - .put("index.codec.compression_level", randomIntBetween(1, 6)) - .build() - ); - ensureGreen(index); - - assertAcked(client().admin().indices().prepareClose(index).setWaitForActiveShards(1)); - - Throwable executionException = expectThrows( - ExecutionException.class, - () -> client().admin() - .indices() - .updateSettings( - new UpdateSettingsRequest(index).settings( - Settings.builder().put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) - ) - ) - .get() - ); - - Throwable rootCause = Throwables.getRootCause(executionException); - assertEquals(IllegalArgumentException.class, rootCause.getClass()); - assertTrue(rootCause.getMessage().startsWith("Compression level cannot be set")); - - assertAcked( - client().admin() - .indices() - .updateSettings( - new UpdateSettingsRequest(index).settings( - Settings.builder() - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) - .put("index.codec.compression_level", (String) null) - ) - ) - .get() - ); - - assertAcked(client().admin().indices().prepareOpen(index).setWaitForActiveShards(1)); - ensureGreen(index); - } - - public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionException, InterruptedException { - - internalCluster().ensureAtLeastNumDataNodes(1); - final String index = "test-index"; - - // creating index - createIndex( - index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) - .build() - ); - ensureGreen(index); - - assertAcked(client().admin().indices().prepareClose(index).setWaitForActiveShards(1)); - - Throwable executionException = expectThrows( - ExecutionException.class, - () -> client().admin() - .indices() - .updateSettings( - new UpdateSettingsRequest(index).settings( - Settings.builder() - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) - .put("index.codec.compression_level", randomIntBetween(1, 6)) - ) - ) - .get() - ); - - Throwable rootCause = Throwables.getRootCause(executionException); - assertEquals(IllegalArgumentException.class, rootCause.getClass()); - assertTrue(rootCause.getMessage().startsWith("Compression level cannot be set")); - - assertAcked( - client().admin() - .indices() - .updateSettings( - new UpdateSettingsRequest(index).settings( - Settings.builder() - .put("index.codec", randomFrom(ZSTD_CODEC, ZSTD_NO_DICT_CODEC)) - .put("index.codec.compression_level", randomIntBetween(1, 6)) - ) - ) - .get() - ); - - assertAcked(client().admin().indices().prepareOpen(index).setWaitForActiveShards(1)); - ensureGreen(index); - } - -} diff --git a/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java b/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java deleted file mode 100644 index bb508282e1952..0000000000000 --- a/plugins/custom-codecs/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec; - -import org.opensearch.action.admin.indices.flush.FlushResponse; -import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.opensearch.action.admin.indices.refresh.RefreshResponse; -import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; -import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.customcodecs.CustomCodecPlugin; -import org.opensearch.index.engine.Segment; -import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toList; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_WRITE; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.is; - -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) -public class MultiCodecMergeIT extends OpenSearchIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singletonList(CustomCodecPlugin.class); - } - - public void testForceMergeMultipleCodecs() throws ExecutionException, InterruptedException { - - Map codecMap = Map.of( - "best_compression", - "BEST_COMPRESSION", - "zlib", - "BEST_COMPRESSION", - "zstd_no_dict", - "ZSTD_NO_DICT", - "zstd", - "ZSTD", - "default", - "BEST_SPEED", - "lz4", - "BEST_SPEED" - ); - - for (Map.Entry codec : codecMap.entrySet()) { - forceMergeMultipleCodecs(codec.getKey(), codec.getValue(), codecMap); - } - - } - - private void forceMergeMultipleCodecs(String finalCodec, String finalCodecMode, Map codecMap) throws ExecutionException, - InterruptedException { - - internalCluster().ensureAtLeastNumDataNodes(1); - final String index = "test-index" + finalCodec; - - // creating index - createIndex( - index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", "default") - .put("index.merge.policy.max_merged_segment", "1b") - .build() - ); - ensureGreen(index); - // ingesting and asserting segment codec mode for all four codecs - for (Map.Entry codec : codecMap.entrySet()) { - useCodec(index, codec.getKey()); - ingestDocs(index); - } - - assertTrue( - getSegments(index).stream() - .flatMap(s -> s.getAttributes().values().stream()) - .collect(Collectors.toSet()) - .containsAll(codecMap.values()) - ); - - // force merge into final codec - useCodec(index, finalCodec); - flushAndRefreshIndex(index); - final ForceMergeResponse forceMergeResponse = client().admin().indices().prepareForceMerge(index).setMaxNumSegments(1).get(); - - assertThat(forceMergeResponse.getFailedShards(), is(0)); - assertThat(forceMergeResponse.getSuccessfulShards(), is(1)); - - flushAndRefreshIndex(index); - - List segments = getSegments(index).stream().filter(Segment::isSearch).collect(Collectors.toList()); - assertEquals(1, segments.size()); - assertTrue(segments.stream().findFirst().get().attributes.containsValue(finalCodecMode)); - } - - private void useCodec(String index, String codec) throws ExecutionException, InterruptedException { - assertAcked(client().admin().indices().prepareClose(index).setWaitForActiveShards(1)); - - assertAcked( - client().admin() - .indices() - .updateSettings(new UpdateSettingsRequest(index).settings(Settings.builder().put("index.codec", codec))) - .get() - ); - - assertAcked(client().admin().indices().prepareOpen(index).setWaitForActiveShards(1)); - } - - private void ingestDocs(String index) throws InterruptedException { - ingest(index); - flushAndRefreshIndex(index); - } - - private ArrayList getSegments(String index) { - - return new ArrayList<>( - client().admin() - .indices() - .segments(new IndicesSegmentsRequest(index)) - .actionGet() - .getIndices() - .get(index) - .getShards() - .get(0) - .getShards()[0].getSegments() - ); - } - - private void ingest(String index) throws InterruptedException { - - final int nbDocs = randomIntBetween(1, 5); - indexRandom( - randomBoolean(), - false, - randomBoolean(), - IntStream.range(0, nbDocs) - .mapToObj(i -> client().prepareIndex(index).setId(UUID.randomUUID().toString()).setSource("num", i)) - .collect(toList()) - ); - } - - private void flushAndRefreshIndex(String index) { - - // Request is not blocked - for (String blockSetting : Arrays.asList( - SETTING_BLOCKS_READ, - SETTING_BLOCKS_WRITE, - SETTING_READ_ONLY, - SETTING_BLOCKS_METADATA, - SETTING_READ_ONLY_ALLOW_DELETE - )) { - try { - enableIndexBlock(index, blockSetting); - FlushResponse flushResponse = client().admin().indices().prepareFlush(index).setForce(true).execute().actionGet(); - assertNoFailures(flushResponse); - RefreshResponse response = client().admin().indices().prepareRefresh(index).execute().actionGet(); - assertNoFailures(response); - } finally { - disableIndexBlock(index, blockSetting); - } - } - } - -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java deleted file mode 100644 index 91a13a1d924a2..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.index.IndexSettings; -import org.opensearch.index.codec.CodecServiceFactory; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.plugins.EnginePlugin; -import org.opensearch.plugins.Plugin; - -import java.util.Optional; - -/** - * A plugin that implements custom codecs. Supports these codecs: - *

    - *
  • ZSTD - *
  • ZSTDNODICT - *
- * - * @opensearch.internal - */ -public final class CustomCodecPlugin extends Plugin implements EnginePlugin { - - /** - * Creates a new instance - */ - public CustomCodecPlugin() {} - - /** - * @param indexSettings is the default indexSettings - * @return the engine factory - */ - @Override - public Optional getCustomCodecServiceFactory(final IndexSettings indexSettings) { - String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); - if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) { - return Optional.of(new CustomCodecServiceFactory()); - } - return Optional.empty(); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java deleted file mode 100644 index de0eb2b3286d3..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.Codec; -import org.opensearch.common.collect.MapBuilder; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.mapper.MapperService; - -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Stream; - -import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; - -/** - * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs. - */ -public class CustomCodecService extends CodecService { - private final Map codecs; - /** - * ZStandard codec - */ - public static final String ZSTD_CODEC = "zstd"; - /** - * ZStandard without dictionary codec - */ - public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; - - /** - * Creates a new CustomCodecService. - * - * @param mapperService The mapper service. - * @param indexSettings The index settings. - * @param logger The logger. - */ - public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) { - super(mapperService, indexSettings, logger); - int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); - final MapBuilder codecs = MapBuilder.newMapBuilder(); - if (mapperService == null) { - codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); - } else { - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); - } - this.codecs = codecs.immutableMap(); - } - - @Override - public Codec codec(String name) { - Codec codec = codecs.get(name); - if (codec == null) { - return super.codec(name); - } - return codec; - } - - @Override - public String[] availableCodecs() { - return Stream.concat(Arrays.stream(super.availableCodecs()), codecs.keySet().stream()).toArray(String[]::new); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java deleted file mode 100644 index d634616162684..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.codec.CodecServiceConfig; -import org.opensearch.index.codec.CodecServiceFactory; - -/** - * A factory for creating new {@link CodecService} instance - */ -public class CustomCodecServiceFactory implements CodecServiceFactory { - - /** Creates a new instance. */ - public CustomCodecServiceFactory() {} - - @Override - public CodecService createCodecService(CodecServiceConfig config) { - return new CustomCodecService(config.getMapperService(), config.getIndexSettings(), config.getLogger()); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java deleted file mode 100644 index 89acc980abd2f..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.FilterCodec; -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; - -import java.util.Collections; -import java.util.Set; - -/** - * - * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. - * Supports two modes zstd and zstd_no_dict. - * - * @opensearch.internal - */ -public abstract class Lucene95CustomCodec extends FilterCodec { - - /** Default compression level used for compression */ - public static final int DEFAULT_COMPRESSION_LEVEL = 3; - - /** Each mode represents a compression algorithm. */ - public enum Mode { - /** - * ZStandard mode with dictionary - */ - ZSTD("ZSTD", Set.of("zstd")), - /** - * ZStandard mode without dictionary - */ - ZSTD_NO_DICT("ZSTDNODICT", Set.of("zstd_no_dict")), - /** - * Deprecated ZStandard mode, added for backward compatibility to support indices created in 2.9.0 where - * both ZSTD and ZSTD_NO_DICT used Lucene95CustomCodec underneath. This should not be used to - * create new indices. - */ - ZSTD_DEPRECATED("Lucene95CustomCodec", Collections.emptySet()); - - private final String codec; - private final Set aliases; - - Mode(String codec, Set aliases) { - this.codec = codec; - this.aliases = aliases; - } - - /** - * Returns the Codec that is registered with Lucene - */ - public String getCodec() { - return codec; - } - - /** - * Returns the aliases of the Codec - */ - public Set getAliases() { - return aliases; - } - } - - private final StoredFieldsFormat storedFieldsFormat; - - /** - * Creates a new compression codec with the default compression level. - * - * @param mode The compression codec (ZSTD or ZSTDNODICT). - */ - public Lucene95CustomCodec(Mode mode) { - this(mode, DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD or ZSTDNODICT). - * @param compressionLevel The compression level. - */ - public Lucene95CustomCodec(Mode mode, int compressionLevel) { - super(mode.getCodec(), new Lucene95Codec()); - this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); - } - - /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD or ZSTDNODICT). - * @param compressionLevel The compression level. - * @param mapperService The mapper service. - * @param logger The logger. - */ - public Lucene95CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); - } - - @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; - } - - @Override - public String toString() { - return getClass().getSimpleName(); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java deleted file mode 100644 index 79d97035089ab..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.StoredFieldsReader; -import org.apache.lucene.codecs.StoredFieldsWriter; -import org.apache.lucene.codecs.compressing.CompressionMode; -import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; - -import java.io.IOException; -import java.util.Objects; - -/** Stored field format used by pluggable codec */ -public class Lucene95CustomStoredFieldsFormat extends StoredFieldsFormat { - - /** A key that we use to map to a mode */ - public static final String MODE_KEY = Lucene95CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; - - private static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; - private static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; - private static final int ZSTD_BLOCK_SHIFT = 10; - - private final CompressionMode zstdCompressionMode; - private final CompressionMode zstdNoDictCompressionMode; - - private final Lucene95CustomCodec.Mode mode; - private final int compressionLevel; - - /** default constructor */ - public Lucene95CustomStoredFieldsFormat() { - this(Lucene95CustomCodec.Mode.ZSTD, Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new instance. - * - * @param mode The mode represents ZSTD or ZSTDNODICT - */ - public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode) { - this(mode, Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new instance with the specified mode and compression level. - * - * @param mode The mode represents ZSTD or ZSTDNODICT - * @param compressionLevel The compression level for the mode. - */ - public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compressionLevel) { - this.mode = Objects.requireNonNull(mode); - this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel); - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); - } - - /** - * Returns a {@link StoredFieldsReader} to load stored fields. - * @param directory The index directory. - * @param si The SegmentInfo that stores segment information. - * @param fn The fieldInfos. - * @param context The IOContext that holds additional details on the merge/search context. - */ - @Override - public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { - String value = si.getAttribute(MODE_KEY); - if (value == null) { - throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); - } - Lucene95CustomCodec.Mode mode = Lucene95CustomCodec.Mode.valueOf(value); - return impl(mode).fieldsReader(directory, si, fn, context); - } - - /** - * Returns a {@link StoredFieldsReader} to write stored fields. - * @param directory The index directory. - * @param si The SegmentInfo that stores segment information. - * @param context The IOContext that holds additional details on the merge/search context. - */ - @Override - public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { - String previous = si.putAttribute(MODE_KEY, mode.name()); - if (previous != null && previous.equals(mode.name()) == false) { - throw new IllegalStateException( - "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() - ); - } - return impl(mode).fieldsWriter(directory, si, context); - } - - StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { - switch (mode) { - case ZSTD: - case ZSTD_DEPRECATED: - return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstd", - zstdCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT - ); - case ZSTD_NO_DICT: - return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstdNoDict", - zstdNoDictCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT - ); - default: - throw new AssertionError(); - } - } - - Lucene95CustomCodec.Mode getMode() { - return mode; - } - - /** - * Returns the compression level. - */ - public int getCompressionLevel() { - return compressionLevel; - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java deleted file mode 100644 index a3e3a34a5d258..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.opensearch.common.settings.Setting; -import org.opensearch.index.codec.CodecAliases; -import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; - -import java.util.Set; - -/** - * ZstdCodec provides ZSTD compressor using the zstd-jni library. - */ -public class ZstdCodec extends Lucene95CustomCodec implements CodecSettings, CodecAliases { - - /** - * Creates a new ZstdCodec instance with the default compression level. - */ - public ZstdCodec() { - this(DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new ZstdCodec instance. - * - * @param compressionLevel The compression level. - */ - public ZstdCodec(int compressionLevel) { - super(Mode.ZSTD, compressionLevel); - } - - /** - * Creates a new ZstdCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public ZstdCodec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD, compressionLevel, mapperService, logger); - } - - /** The name for this codec. */ - @Override - public String toString() { - return getClass().getSimpleName(); - } - - @Override - public boolean supports(Setting setting) { - return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); - } - - @Override - public Set aliases() { - return Mode.ZSTD.getAliases(); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java deleted file mode 100644 index 05ff725933e1a..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdCompressCtx; -import com.github.luben.zstd.ZstdDecompressCtx; -import com.github.luben.zstd.ZstdDictCompress; -import com.github.luben.zstd.ZstdDictDecompress; - -import org.apache.lucene.codecs.compressing.CompressionMode; -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; -import org.apache.lucene.store.ByteBuffersDataInput; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; - -/** Zstandard Compression Mode */ -public class ZstdCompressionMode extends CompressionMode { - - private static final int NUM_SUB_BLOCKS = 10; - private static final int DICT_SIZE_FACTOR = 6; - private static final int DEFAULT_COMPRESSION_LEVEL = 6; - - private final int compressionLevel; - - /** default constructor */ - protected ZstdCompressionMode() { - this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; - } - - /** - * Creates a new instance. - * - * @param compressionLevel The compression level to use. - */ - protected ZstdCompressionMode(int compressionLevel) { - this.compressionLevel = compressionLevel; - } - - /** Creates a new compressor instance.*/ - @Override - public Compressor newCompressor() { - return new ZstdCompressor(compressionLevel); - } - - /** Creates a new decompressor instance. */ - @Override - public Decompressor newDecompressor() { - return new ZstdDecompressor(); - } - - /** zstandard compressor */ - private static final class ZstdCompressor extends Compressor { - - private final int compressionLevel; - private byte[] compressedBuffer; - - /** compressor with a given compresion level */ - public ZstdCompressor(int compressionLevel) { - this.compressionLevel = compressionLevel; - compressedBuffer = BytesRef.EMPTY_BYTES; - } - - /*resuable compress function*/ - private void doCompress(byte[] bytes, int offset, int length, ZstdCompressCtx cctx, DataOutput out) throws IOException { - if (length == 0) { - out.writeVInt(0); - return; - } - final int maxCompressedLength = (int) Zstd.compressBound(length); - compressedBuffer = ArrayUtil.growNoCopy(compressedBuffer, maxCompressedLength); - - int compressedSize = cctx.compressByteArray(compressedBuffer, 0, compressedBuffer.length, bytes, offset, length); - - out.writeVInt(compressedSize); - out.writeBytes(compressedBuffer, compressedSize); - } - - private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { - assert offset >= 0 : "offset value must be greater than 0"; - - final int dictLength = length / (NUM_SUB_BLOCKS * DICT_SIZE_FACTOR); - final int blockLength = (length - dictLength + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS; - out.writeVInt(dictLength); - out.writeVInt(blockLength); - - final int end = offset + length; - assert end >= 0 : "buffer read size must be greater than 0"; - - try (ZstdCompressCtx cctx = new ZstdCompressCtx()) { - cctx.setLevel(compressionLevel); - - // dictionary compression first - doCompress(bytes, offset, dictLength, cctx, out); - try (ZstdDictCompress dictCompress = new ZstdDictCompress(bytes, offset, dictLength, compressionLevel)) { - cctx.loadDict(dictCompress); - - for (int start = offset + dictLength; start < end; start += blockLength) { - int l = Math.min(blockLength, end - start); - doCompress(bytes, start, l, cctx, out); - } - } - } - } - - @Override - public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); - byte[] bytes = new byte[length]; - buffersInput.readBytes(bytes, 0, length); - compress(bytes, 0, length, out); - } - - @Override - public void close() throws IOException {} - } - - /** zstandard decompressor */ - private static final class ZstdDecompressor extends Decompressor { - - private byte[] compressedBuffer; - - /** default decompressor */ - public ZstdDecompressor() { - compressedBuffer = BytesRef.EMPTY_BYTES; - } - - /*resuable decompress function*/ - private void doDecompress(DataInput in, ZstdDecompressCtx dctx, BytesRef bytes, int decompressedLen) throws IOException { - final int compressedLength = in.readVInt(); - if (compressedLength == 0) { - return; - } - - compressedBuffer = ArrayUtil.growNoCopy(compressedBuffer, compressedLength); - in.readBytes(compressedBuffer, 0, compressedLength); - - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + decompressedLen); - int uncompressed = dctx.decompressByteArray(bytes.bytes, bytes.length, decompressedLen, compressedBuffer, 0, compressedLength); - - if (decompressedLen != uncompressed) { - throw new IllegalStateException(decompressedLen + " " + uncompressed); - } - bytes.length += uncompressed; - } - - @Override - public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { - assert offset + length <= originalLength : "buffer read size must be within limit"; - - if (length == 0) { - bytes.length = 0; - return; - } - final int dictLength = in.readVInt(); - final int blockLength = in.readVInt(); - bytes.bytes = ArrayUtil.growNoCopy(bytes.bytes, dictLength); - bytes.offset = bytes.length = 0; - - try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { - - // decompress dictionary first - doDecompress(in, dctx, bytes, dictLength); - try (ZstdDictDecompress dictDecompress = new ZstdDictDecompress(bytes.bytes, 0, dictLength)) { - dctx.loadDict(dictDecompress); - - int offsetInBlock = dictLength; - int offsetInBytesRef = offset; - - // Skip unneeded blocks - while (offsetInBlock + blockLength < offset) { - final int compressedLength = in.readVInt(); - in.skipBytes(compressedLength); - offsetInBlock += blockLength; - offsetInBytesRef -= blockLength; - } - - // Read blocks that intersect with the interval we need - while (offsetInBlock < offset + length) { - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength); - int l = Math.min(blockLength, originalLength - offsetInBlock); - doDecompress(in, dctx, bytes, l); - offsetInBlock += blockLength; - } - - bytes.offset = offsetInBytesRef; - bytes.length = length; - - assert bytes.isValid() : "decompression output is corrupted"; - } - } - } - - @Override - public Decompressor clone() { - return new ZstdDecompressor(); - } - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java deleted file mode 100644 index 02fa386db97b3..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.opensearch.common.settings.Setting; -import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; - -/** - * ZstdDeprecatedCodec provides ZSTD compressor using the zstd-jni library. - * Added to support backward compatibility for indices created with Lucene95CustomCodec as codec name. - */ -@Deprecated(since = "2.10") -public class ZstdDeprecatedCodec extends Lucene95CustomCodec implements CodecSettings { - - /** - * Creates a new ZstdDefaultCodec instance with the default compression level. - */ - public ZstdDeprecatedCodec() { - this(DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new ZstdDefaultCodec instance. - * - * @param compressionLevel The compression level. - */ - public ZstdDeprecatedCodec(int compressionLevel) { - super(Mode.ZSTD_DEPRECATED, compressionLevel); - } - - /** - * Creates a new ZstdDefaultCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public ZstdDeprecatedCodec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD_DEPRECATED, compressionLevel, mapperService, logger); - } - - /** The name for this codec. */ - @Override - public String toString() { - return getClass().getSimpleName(); - } - - @Override - public boolean supports(Setting setting) { - return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java deleted file mode 100644 index ea7351f755361..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.opensearch.common.settings.Setting; -import org.opensearch.index.codec.CodecAliases; -import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; - -import java.util.Set; - -/** - * ZstdNoDictCodec provides ZSTD compressor without a dictionary support. - */ -public class ZstdNoDictCodec extends Lucene95CustomCodec implements CodecSettings, CodecAliases { - - /** - * Creates a new ZstdNoDictCodec instance with the default compression level. - */ - public ZstdNoDictCodec() { - this(DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new ZstdNoDictCodec instance. - * - * @param compressionLevel The compression level. - */ - public ZstdNoDictCodec(int compressionLevel) { - super(Mode.ZSTD_NO_DICT, compressionLevel); - } - - /** - * Creates a new ZstdNoDictCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public ZstdNoDictCodec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); - } - - /** The name for this codec. */ - @Override - public String toString() { - return getClass().getSimpleName(); - } - - @Override - public boolean supports(Setting setting) { - return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); - } - - @Override - public Set aliases() { - return Mode.ZSTD_NO_DICT.getAliases(); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java deleted file mode 100644 index af4e92b78ed0f..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import com.github.luben.zstd.Zstd; - -import org.apache.lucene.codecs.compressing.CompressionMode; -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; -import org.apache.lucene.store.ByteBuffersDataInput; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; - -/** ZSTD Compression Mode (without a dictionary support). */ -public class ZstdNoDictCompressionMode extends CompressionMode { - - private static final int NUM_SUB_BLOCKS = 10; - private static final int DEFAULT_COMPRESSION_LEVEL = 6; - - private final int compressionLevel; - - /** default constructor */ - protected ZstdNoDictCompressionMode() { - this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; - } - - /** - * Creates a new instance with the given compression level. - * - * @param compressionLevel The compression level. - */ - protected ZstdNoDictCompressionMode(int compressionLevel) { - this.compressionLevel = compressionLevel; - } - - /** Creates a new compressor instance.*/ - @Override - public Compressor newCompressor() { - return new ZstdCompressor(compressionLevel); - } - - /** Creates a new decompressor instance. */ - @Override - public Decompressor newDecompressor() { - return new ZstdDecompressor(); - } - - /** zstandard compressor */ - private static final class ZstdCompressor extends Compressor { - - private final int compressionLevel; - private byte[] compressedBuffer; - - /** compressor with a given compresion level */ - public ZstdCompressor(int compressionLevel) { - this.compressionLevel = compressionLevel; - compressedBuffer = BytesRef.EMPTY_BYTES; - } - - private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { - assert offset >= 0 : "offset value must be greater than 0"; - - int blockLength = (length + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS; - out.writeVInt(blockLength); - - final int end = offset + length; - assert end >= 0 : "buffer read size must be greater than 0"; - - for (int start = offset; start < end; start += blockLength) { - int l = Math.min(blockLength, end - start); - - if (l == 0) { - out.writeVInt(0); - return; - } - - final int maxCompressedLength = (int) Zstd.compressBound(l); - compressedBuffer = ArrayUtil.growNoCopy(compressedBuffer, maxCompressedLength); - - int compressedSize = (int) Zstd.compressByteArray( - compressedBuffer, - 0, - compressedBuffer.length, - bytes, - start, - l, - compressionLevel - ); - - out.writeVInt(compressedSize); - out.writeBytes(compressedBuffer, compressedSize); - } - } - - @Override - public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); - byte[] bytes = new byte[length]; - buffersInput.readBytes(bytes, 0, length); - compress(bytes, 0, length, out); - } - - @Override - public void close() throws IOException {} - } - - /** zstandard decompressor */ - private static final class ZstdDecompressor extends Decompressor { - - private byte[] compressed; - - /** default decompressor */ - public ZstdDecompressor() { - compressed = BytesRef.EMPTY_BYTES; - } - - @Override - public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { - assert offset + length <= originalLength : "buffer read size must be within limit"; - - if (length == 0) { - bytes.length = 0; - return; - } - - final int blockLength = in.readVInt(); - bytes.offset = bytes.length = 0; - int offsetInBlock = 0; - int offsetInBytesRef = offset; - - // Skip unneeded blocks - while (offsetInBlock + blockLength < offset) { - final int compressedLength = in.readVInt(); - in.skipBytes(compressedLength); - offsetInBlock += blockLength; - offsetInBytesRef -= blockLength; - } - - // Read blocks that intersect with the interval we need - while (offsetInBlock < offset + length) { - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength); - final int compressedLength = in.readVInt(); - if (compressedLength == 0) { - return; - } - compressed = ArrayUtil.growNoCopy(compressed, compressedLength); - in.readBytes(compressed, 0, compressedLength); - - int l = Math.min(blockLength, originalLength - offsetInBlock); - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + l); - - byte[] output = new byte[l]; - - final int uncompressed = (int) Zstd.decompressByteArray(output, 0, l, compressed, 0, compressedLength); - System.arraycopy(output, 0, bytes.bytes, bytes.length, uncompressed); - - bytes.length += uncompressed; - offsetInBlock += blockLength; - } - - bytes.offset = offsetInBytesRef; - bytes.length = length; - - assert bytes.isValid() : "decompression output is corrupted."; - } - - @Override - public Decompressor clone() { - return new ZstdDecompressor(); - } - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java deleted file mode 100644 index e996873963b1b..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * A plugin that implements compression codecs with native implementation. - */ -package org.opensearch.index.codec.customcodecs; diff --git a/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy b/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy deleted file mode 100644 index 8161010cfa897..0000000000000 --- a/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy +++ /dev/null @@ -1,11 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -grant codeBase "${codebase.zstd-jni}" { - permission java.lang.RuntimePermission "loadLibrary.*"; -}; diff --git a/plugins/custom-codecs/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/plugins/custom-codecs/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec deleted file mode 100644 index ba5054055d00a..0000000000000 --- a/plugins/custom-codecs/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ /dev/null @@ -1,3 +0,0 @@ -org.opensearch.index.codec.customcodecs.ZstdCodec -org.opensearch.index.codec.customcodecs.ZstdNoDictCodec -org.opensearch.index.codec.customcodecs.ZstdDeprecatedCodec diff --git a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java deleted file mode 100644 index cc794eb2c48f1..0000000000000 --- a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; -import org.apache.lucene.store.ByteArrayDataInput; -import org.apache.lucene.store.ByteBuffersDataInput; -import org.apache.lucene.store.ByteBuffersDataOutput; -import org.apache.lucene.tests.util.LineFileDocs; -import org.apache.lucene.tests.util.TestUtil; -import org.apache.lucene.util.BytesRef; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Random; - -/** - * Test cases for compressors (based on {@See org.opensearch.common.compress.DeflateCompressTests}). - */ -public abstract class AbstractCompressorTests extends OpenSearchTestCase { - - abstract Compressor compressor(); - - abstract Decompressor decompressor(); - - public void testEmpty() throws IOException { - final byte[] bytes = "".getBytes(StandardCharsets.UTF_8); - doTest(bytes); - } - - public void testShortLiterals() throws IOException { - final byte[] bytes = "1234567345673456745608910123".getBytes(StandardCharsets.UTF_8); - doTest(bytes); - } - - public void testRandom() throws IOException { - Random r = random(); - for (int i = 0; i < 10; i++) { - final byte[] bytes = new byte[TestUtil.nextInt(r, 1, 100000)]; - r.nextBytes(bytes); - doTest(bytes); - } - } - - public void testLineDocs() throws IOException { - Random r = random(); - LineFileDocs lineFileDocs = new LineFileDocs(r); - for (int i = 0; i < 10; i++) { - int numDocs = TestUtil.nextInt(r, 1, 200); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - for (int j = 0; j < numDocs; j++) { - String s = lineFileDocs.nextDoc().get("body"); - bos.write(s.getBytes(StandardCharsets.UTF_8)); - } - doTest(bos.toByteArray()); - } - lineFileDocs.close(); - } - - public void testRepetitionsL() throws IOException { - Random r = random(); - for (int i = 0; i < 10; i++) { - int numLongs = TestUtil.nextInt(r, 1, 10000); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - long theValue = r.nextLong(); - for (int j = 0; j < numLongs; j++) { - if (r.nextInt(10) == 0) { - theValue = r.nextLong(); - } - bos.write((byte) (theValue >>> 56)); - bos.write((byte) (theValue >>> 48)); - bos.write((byte) (theValue >>> 40)); - bos.write((byte) (theValue >>> 32)); - bos.write((byte) (theValue >>> 24)); - bos.write((byte) (theValue >>> 16)); - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - doTest(bos.toByteArray()); - } - } - - public void testRepetitionsI() throws IOException { - Random r = random(); - for (int i = 0; i < 10; i++) { - int numInts = TestUtil.nextInt(r, 1, 20000); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - int theValue = r.nextInt(); - for (int j = 0; j < numInts; j++) { - if (r.nextInt(10) == 0) { - theValue = r.nextInt(); - } - bos.write((byte) (theValue >>> 24)); - bos.write((byte) (theValue >>> 16)); - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - doTest(bos.toByteArray()); - } - } - - public void testRepetitionsS() throws IOException { - Random r = random(); - for (int i = 0; i < 10; i++) { - int numShorts = TestUtil.nextInt(r, 1, 40000); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - short theValue = (short) r.nextInt(65535); - for (int j = 0; j < numShorts; j++) { - if (r.nextInt(10) == 0) { - theValue = (short) r.nextInt(65535); - } - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - doTest(bos.toByteArray()); - } - } - - public void testMixed() throws IOException { - Random r = random(); - LineFileDocs lineFileDocs = new LineFileDocs(r); - for (int i = 0; i < 2; ++i) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - int prevInt = r.nextInt(); - long prevLong = r.nextLong(); - while (bos.size() < 400000) { - switch (r.nextInt(4)) { - case 0: - addInt(r, prevInt, bos); - break; - case 1: - addLong(r, prevLong, bos); - break; - case 2: - addString(lineFileDocs, bos); - break; - case 3: - addBytes(r, bos); - break; - default: - throw new IllegalStateException("Random is broken"); - } - } - doTest(bos.toByteArray()); - } - } - - private void addLong(Random r, long prev, ByteArrayOutputStream bos) { - long theValue = prev; - if (r.nextInt(10) != 0) { - theValue = r.nextLong(); - } - bos.write((byte) (theValue >>> 56)); - bos.write((byte) (theValue >>> 48)); - bos.write((byte) (theValue >>> 40)); - bos.write((byte) (theValue >>> 32)); - bos.write((byte) (theValue >>> 24)); - bos.write((byte) (theValue >>> 16)); - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - - private void addInt(Random r, int prev, ByteArrayOutputStream bos) { - int theValue = prev; - if (r.nextInt(10) != 0) { - theValue = r.nextInt(); - } - bos.write((byte) (theValue >>> 24)); - bos.write((byte) (theValue >>> 16)); - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - - private void addString(LineFileDocs lineFileDocs, ByteArrayOutputStream bos) throws IOException { - String s = lineFileDocs.nextDoc().get("body"); - bos.write(s.getBytes(StandardCharsets.UTF_8)); - } - - private void addBytes(Random r, ByteArrayOutputStream bos) throws IOException { - byte bytes[] = new byte[TestUtil.nextInt(r, 1, 10000)]; - r.nextBytes(bytes); - bos.write(bytes); - } - - private void doTest(byte[] bytes) throws IOException { - final int length = bytes.length; - - ByteBuffersDataInput in = new ByteBuffersDataInput(List.of(ByteBuffer.wrap(bytes))); - ByteBuffersDataOutput out = new ByteBuffersDataOutput(); - - // let's compress - Compressor compressor = compressor(); - compressor.compress(in, out); - byte[] compressed = out.toArrayCopy(); - - // let's decompress - BytesRef outbytes = new BytesRef(); - Decompressor decompressor = decompressor(); - decompressor.decompress(new ByteArrayDataInput(compressed), length, 0, length, outbytes); - - // get the uncompressed array out of outbytes - byte[] restored = new byte[outbytes.length]; - System.arraycopy(outbytes.bytes, 0, restored, 0, outbytes.length); - - assertArrayEquals(bytes, restored); - } - -} diff --git a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java deleted file mode 100644 index 5365b9e222d9a..0000000000000 --- a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.LogManager; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; -import org.opensearch.common.settings.IndexScopedSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.analysis.IndexAnalyzers; -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.codec.CodecServiceConfig; -import org.opensearch.index.codec.CodecServiceFactory; -import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.similarity.SimilarityService; -import org.opensearch.indices.mapper.MapperRegistry; -import org.opensearch.plugins.MapperPlugin; -import org.opensearch.test.IndexSettingsModule; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; - -import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; -import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; -import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; - -@SuppressCodecs("*") // we test against default codec so never get a random one here! -public class CustomCodecTests extends OpenSearchTestCase { - - private CustomCodecPlugin plugin; - - @Before - public void setup() { - plugin = new CustomCodecPlugin(); - } - - public void testZstd() throws Exception { - Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDict() throws Exception { - Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdDeprecatedCodec() { - final IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> createCodecService(false).codec("ZSTD_DEPRECATED") - ); - assertTrue(e.getMessage().startsWith("failed to find codec")); - } - - public void testZstdWithCompressionLevel() throws Exception { - int randomCompressionLevel = randomIntBetween(1, 6); - Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDictWithCompressionLevel() throws Exception { - int randomCompressionLevel = randomIntBetween(1, 6); - Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); - } - - public void testBestCompressionWithCompressionLevel() { - final Settings zstdSettings = Settings.builder() - .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(ZSTD_CODEC, ZSTD_NO_DICT_CODEC)) - .build(); - - // able to validate zstd - final IndexScopedSettings zstdIndexScopedSettings = new IndexScopedSettings( - zstdSettings, - IndexScopedSettings.BUILT_IN_INDEX_SETTINGS - ); - zstdIndexScopedSettings.validate(zstdSettings, true); - } - - public void testLuceneCodecsWithCompressionLevel() { - final Settings customCodecSettings = Settings.builder() - .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom("zstd", "zstd_no_dict")) - .build(); - - final IndexScopedSettings customCodecIndexScopedSettings = new IndexScopedSettings( - customCodecSettings, - IndexScopedSettings.BUILT_IN_INDEX_SETTINGS - ); - customCodecIndexScopedSettings.validate(customCodecSettings, true); - } - - public void testZstandardCompressionLevelSupport() throws Exception { - CodecService codecService = createCodecService(false); - CodecSettings zstdCodec = (CodecSettings) codecService.codec("zstd"); - CodecSettings zstdNoDictCodec = (CodecSettings) codecService.codec("zstd_no_dict"); - assertTrue(zstdCodec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); - assertTrue(zstdNoDictCodec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); - } - - public void testDefaultMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); - } - - public void testBestCompressionMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); - } - - public void testZstdMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDictMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { - SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); - assertNotNull(v); - assertEquals(expected, Lucene95Codec.Mode.valueOf(v)); - } - - private void assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode expected, Codec actual) throws Exception { - SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene95CustomStoredFieldsFormat.MODE_KEY); - assertNotNull(v); - assertEquals(expected, Lucene95CustomCodec.Mode.valueOf(v)); - } - - private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { - Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); - if (isMapperServiceNull) { - return new CustomCodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test")); - } - return buildCodecService(nodeSettings); - } - - private CodecService createCodecService(int randomCompressionLevel, String codec) throws IOException { - Settings nodeSettings = Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put("index.codec", codec) - .put("index.codec.compression_level", randomCompressionLevel) - .build(); - return buildCodecService(nodeSettings); - } - - private CodecService buildCodecService(Settings nodeSettings) throws IOException { - - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); - SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); - IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; - MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); - MapperService service = new MapperService( - indexSettings, - indexAnalyzers, - xContentRegistry(), - similarityService, - mapperRegistry, - () -> null, - () -> false, - null - ); - - Optional customCodecServiceFactory = plugin.getCustomCodecServiceFactory(indexSettings); - if (customCodecServiceFactory.isPresent()) { - return customCodecServiceFactory.get().createCodecService(new CodecServiceConfig(indexSettings, service, logger)); - } - return new CustomCodecService(service, indexSettings, LogManager.getLogger("test")); - } - - private SegmentReader getSegmentReader(Codec codec) throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(null); - iwc.setCodec(codec); - IndexWriter iw = new IndexWriter(dir, iwc); - iw.addDocument(new Document()); - iw.commit(); - iw.close(); - DirectoryReader ir = DirectoryReader.open(dir); - SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); - ir.close(); - dir.close(); - return sr; - } - -} diff --git a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java deleted file mode 100644 index e87fb56770e4c..0000000000000 --- a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.test.OpenSearchTestCase; - -public class Lucene95CustomStoredFieldsFormatTests extends OpenSearchTestCase { - - public void testDefaultLucene95CustomCodecMode() { - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(); - assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); - } - - public void testZstdNoDictLucene95CustomCodecMode() { - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT - ); - assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); - } - - public void testZstdModeWithCompressionLevel() { - int randomCompressionLevel = randomIntBetween(1, 6); - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD, - randomCompressionLevel - ); - assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); - assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { - int randomCompressionLevel = randomIntBetween(1, 6); - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT, - randomCompressionLevel - ); - assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); - assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); - } - -} diff --git a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java deleted file mode 100644 index 78cf62c08f889..0000000000000 --- a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.codec.customcodecs; - -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; - -/** - * Test ZSTD compression (with dictionary enabled) - */ -public class ZstdCompressorTests extends AbstractCompressorTests { - - private final Compressor compressor = new ZstdCompressionMode().newCompressor(); - private final Decompressor decompressor = new ZstdCompressionMode().newDecompressor(); - - @Override - Compressor compressor() { - return compressor; - } - - @Override - Decompressor decompressor() { - return decompressor; - } -} diff --git a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java deleted file mode 100644 index 2eda81a6af2ab..0000000000000 --- a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.codec.customcodecs; - -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; - -/** - * Test ZSTD compression (with no dictionary). - */ -public class ZstdNoDictCompressorTests extends AbstractCompressorTests { - - private final Compressor compressor = new ZstdNoDictCompressionMode().newCompressor(); - private final Decompressor decompressor = new ZstdNoDictCompressionMode().newDecompressor(); - - @Override - Compressor compressor() { - return compressor; - } - - @Override - Decompressor decompressor() { - return decompressor; - } -} From 5c3d26d62ce7a321e7aebd322f2feda43f14755d Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 8 Sep 2023 14:17:12 -0700 Subject: [PATCH 006/388] Fix ReadContextListenerTests to avoid inconsistent WindowsFS file handle simulation (#9915) Signed-off-by: Kunal Kotwani --- .../stream/read/listener/ReadContextListenerTests.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java index ed08a436d3ca0..21b7b47390a9b 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java @@ -8,6 +8,7 @@ package org.opensearch.common.blobstore.stream.read.listener; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems; import org.opensearch.action.LatchedActionListener; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -32,6 +33,12 @@ import static org.opensearch.common.blobstore.stream.read.listener.ListenerTestUtils.CountingCompletionListener; +/* + WindowsFS tries to simulate file handles in a best case simulation. + The deletion for the open file on an actual Windows system will be performed as soon as the last handle + is closed, which this simulation does not account for. Preventing use of WindowsFS for these tests. + */ +@SuppressFileSystems("WindowsFS") public class ReadContextListenerTests extends OpenSearchTestCase { private Path path; @@ -70,7 +77,6 @@ public void testReadContextListener() throws InterruptedException, IOException { assertEquals(NUMBER_OF_PARTS * PART_SIZE, Files.size(fileLocation)); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9776") public void testReadContextListenerFailure() throws Exception { Path fileLocation = path.resolve(UUID.randomUUID().toString()); List blobPartStreams = initializeBlobPartStreams(); @@ -100,7 +106,7 @@ public int available() { readContextListener.onResponse(readContext); countDownLatch.await(); - assertBusy(() -> { assertFalse(Files.exists(fileLocation)); }); + assertFalse(Files.exists(fileLocation)); } public void testReadContextListenerException() { From 0a0a42eb3c7de580a1c31aa4aa57ead08e622ca4 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 8 Sep 2023 16:52:02 -0700 Subject: [PATCH 007/388] Add release notes for 2.10 (#9955) * Clear CHANGELOG for released 2.x changes Signed-off-by: Kunal Kotwani * Add release notes for 2.10 Signed-off-by: Kunal Kotwani --------- Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 122 +--------------- .../opensearch.release-notes-2.10.0.md | 136 ++++++++++++++++++ 2 files changed, 137 insertions(+), 121 deletions(-) create mode 100644 release-notes/opensearch.release-notes-2.10.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2404c57ee348d..ec6ea2f79f22c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107)) - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679)) -- Added encryption-sdk lib to provide encryption and decryption capabilities ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466) [#9289](https://github.com/opensearch-project/OpenSearch/pull/9289)) -- Added crypto-kms plugin to provide AWS KMS based key providers for encryption/decryption. ([#8465](https://github.com/opensearch-project/OpenSearch/pull/8465)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 @@ -42,7 +40,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) -- Add Encryption SDK dependencies ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) @@ -50,10 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459)) - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) -- [Remote Store] Add Segment download stats to remotestore stats API ([#8718](https://github.com/opensearch-project/OpenSearch/pull/8718)) -- [Remote Store] Add remote segment transfer stats on NodesStats API ([#9168](https://github.com/opensearch-project/OpenSearch/pull/9168) [#9393](https://github.com/opensearch-project/OpenSearch/pull/9393) [#9454](https://github.com/opensearch-project/OpenSearch/pull/9454)) - Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855)) -- [Remote Store] Removing feature flag to mark feature GA ([#9761](https://github.com/opensearch-project/OpenSearch/pull/9761)) ### Deprecated @@ -81,113 +75,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583) -- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221](https://github.com/opensearch-project/OpenSearch/pull/8221)) -- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195](https://github.com/opensearch-project/OpenSearch/pull/8195)) -- Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606)) -- Disallow compression level to be set for default and best_compression index codecs ([#8737](https://github.com/opensearch-project/OpenSearch/pull/8737)) -- Prioritize replica shard movement during shard relocation ([#8875](https://github.com/opensearch-project/OpenSearch/pull/8875)) -- Introducing Default and Best Compression codecs as their algorithm name ([#9123](https://github.com/opensearch-project/OpenSearch/pull/9123)) -- Make SearchTemplateRequest implement IndicesRequest.Replaceable ([#9122](https://github.com/opensearch-project/OpenSearch/pull/9122)) -- [BWC and API enforcement] Define the initial set of annotations, their meaning and relations between them ([#9223](https://github.com/opensearch-project/OpenSearch/pull/9223)) -- [Segment Replication] Support realtime reads for GET requests ([#9212](https://github.com/opensearch-project/OpenSearch/pull/9212)) -- [Feature] Expose term frequency in Painless script score context ([#9081](https://github.com/opensearch-project/OpenSearch/pull/9081)) -- Add support for reading partial files to HDFS repository ([#9513](https://github.com/opensearch-project/OpenSearch/issues/9513)) -- Add support for extensions to search responses using SearchExtBuilder ([#9379](https://github.com/opensearch-project/OpenSearch/pull/9379)) -- [Remote State] Create service to publish cluster state to remote store ([#9160](https://github.com/opensearch-project/OpenSearch/pull/9160)) -- [BWC and API enforcement] Decorate the existing APIs with proper annotations (part 1) ([#9520](https://github.com/opensearch-project/OpenSearch/pull/9520)) -- Add concurrent segment search related metrics to node and index stats ([#9622](https://github.com/opensearch-project/OpenSearch/issues/9622)) -- Decouple replication lag from logic to fail stale replicas ([#9507](https://github.com/opensearch-project/OpenSearch/pull/9507)) -- Expose DelimitedTermFrequencyTokenFilter to allow providing term frequencies along with terms ([#9479](https://github.com/opensearch-project/OpenSearch/pull/9479)) -- APIs for performing async blob reads and async downloads from the repository using multiple streams ([#9592](https://github.com/opensearch-project/OpenSearch/issues/9592)) -- Introduce cluster default remote translog buffer interval setting ([#9584](https://github.com/opensearch-project/OpenSearch/pull/9584)) -- Add average concurrency metric for concurrent segment search ([#9670](https://github.com/opensearch-project/OpenSearch/issues/9670)) -- [Remote state] Integrate remote cluster state in publish/commit flow ([#9665](https://github.com/opensearch-project/OpenSearch/pull/9665)) -- [Segment Replication] Adding segment replication statistics rolled up at index, node and cluster level ([#9709](https://github.com/opensearch-project/OpenSearch/pull/9709)) -- [Remote Store] Changes to introduce repository registration during bootstrap via node attributes. ([#9105](https://github.com/opensearch-project/OpenSearch/pull/9105)) -- [Remote state] Auto restore index metadata from last known cluster state ([#9831](https://github.com/opensearch-project/OpenSearch/pull/9831)) ### Dependencies -- Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) -- Bump `io.grpc:grpc-context` from 1.46.0 to 1.57.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726), [#9145](https://github.com/opensearch-project/OpenSearch/pull/9145)) -- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724)) -- Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) -- Bump `org.apache.zookeeper:zookeeper` from 3.8.1 to 3.9.0 ([#8844](https://github.com/opensearch-project/OpenSearch/pull/8844), [#9146](https://github.com/opensearch-project/OpenSearch/pull/9146)) -- Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842)) -- Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) -- Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) -- OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868) -- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995)) -- Bump `com.gradle.enterprise` from 3.13.3 to 3.14.1 ([#8996](https://github.com/opensearch-project/OpenSearch/pull/8996)) -- Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 ([#8995](https://github.com/opensearch-project/OpenSearch/pull/8995), [#9298](https://github.com/opensearch-project/OpenSearch/pull/9298)) -- Bump `com.google.cloud:google-cloud-core-http` from 2.21.0 to 2.21.1 ([#8999](https://github.com/opensearch-project/OpenSearch/pull/8999)) -- Bump `com.maxmind.geoip2:geoip2` from 4.0.1 to 4.1.0 ([#8998](https://github.com/opensearch-project/OpenSearch/pull/8998)) -- Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 in /plugins/repository-hdfs ([#8997](https://github.com/opensearch-project/OpenSearch/pull/8997)) -- Bump `netty` from 4.1.94.Final to 4.1.96.Final ([#9030](https://github.com/opensearch-project/OpenSearch/pull/9030)) -- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.8 to 1.1.9 ([#9147](https://github.com/opensearch-project/OpenSearch/pull/9147)) -- Bump `org.apache.maven:maven-model` from 3.9.3 to 3.9.4 ([#9148](https://github.com/opensearch-project/OpenSearch/pull/9148)) -- Bump `com.azure:azure-storage-blob` from 12.22.3 to 12.23.0 ([#9231](https://github.com/opensearch-project/OpenSearch/pull/9231)) -- Bump `com.diffplug.spotless` from 6.19.0 to 6.20.0 ([#9227](https://github.com/opensearch-project/OpenSearch/pull/9227)) -- Bump `org.xerial.snappy:snappy-java` from 1.1.8.2 to 1.1.10.3 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) -- Bump `com.squareup.okhttp3:okhttp` from 4.9.3 to 4.11.0 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) -- Bump `com.squareup.okio:okio` from 2.8.0 to 3.5.0 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) -- Bump `com.google.code.gson:gson` from 2.9.0 to 2.10.1 ([#9230](https://github.com/opensearch-project/OpenSearch/pull/9230)) -- Bump `lycheeverse/lychee-action` from 1.2.0 to 1.8.0 ([#9228](https://github.com/opensearch-project/OpenSearch/pull/9228)) -- Bump `snakeyaml` from 2.0 to 2.1 ([#9269](https://github.com/opensearch-project/OpenSearch/pull/9269)) -- Bump `aws-actions/configure-aws-credentials` from 1 to 2 ([#9302](https://github.com/opensearch-project/OpenSearch/pull/9302)) -- Bump `com.github.luben:zstd-jni` from 1.5.5-3 to 1.5.5-5 ([#9431](https://github.com/opensearch-project/OpenSearch/pull/9431) -- Bump `actions/setup-java` from 2 to 3 ([#9457](https://github.com/opensearch-project/OpenSearch/pull/9457)) -- Bump `com.google.api:gax` from 2.27.0 to 2.32.0 ([#9300](https://github.com/opensearch-project/OpenSearch/pull/9300)) -- Bump `netty` from 4.1.96.Final to 4.1.97.Final ([#9553](https://github.com/opensearch-project/OpenSearch/pull/9553)) -- Bump `io.grpc:grpc-api` from 1.57.1 to 1.57.2 ([#9578](https://github.com/opensearch-project/OpenSearch/pull/9578)) -- Bump `org.apache.ant:ant` from 1.10.13 to 1.10.14 ([#9579](https://github.com/opensearch-project/OpenSearch/pull/9579)) ### Changed -- Default to mmapfs within hybridfs ([#8508](https://github.com/opensearch-project/OpenSearch/pull/8508)) -- Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) -- Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) -- Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) -- [Refactor] StreamIO from common to core.common namespace in core lib ([#8157](https://github.com/opensearch-project/OpenSearch/pull/8157)) -- [Refactor] Remaining HPPC to java.util collections ([#8730](https://github.com/opensearch-project/OpenSearch/pull/8730)) -- Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) -- Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) -- Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805)) -- Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807)) -- Allow test clusters to run with TLS ([#8900](https://github.com/opensearch-project/OpenSearch/pull/8900)) -- Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792)) -- [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792)) -- Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) -- Performance improvements for BytesRefHash ([#8788](https://github.com/opensearch-project/OpenSearch/pull/8788)) -- Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801)) -- [Remove] Deprecated Fractional ByteSizeValue support #9005 ([#9005](https://github.com/opensearch-project/OpenSearch/pull/9005)) -- Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801)) -- [Remote Store] Restrict user override for remote store index level settings ([#8812](https://github.com/opensearch-project/OpenSearch/pull/8812)) -- [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636)) -- Make MultiBucketConsumerService thread safe to use across slices during search ([#9047](https://github.com/opensearch-project/OpenSearch/pull/9047)) -- Removed blocking wait in TransportGetSnapshotsAction which was exhausting generic threadpool ([#8377](https://github.com/opensearch-project/OpenSearch/pull/8377)) -- Adds support for tracing runnable scenarios ([#8831](https://github.com/opensearch-project/OpenSearch/pull/8831)) -- Change shard_size and shard_min_doc_count evaluation to happen in shard level reduce phase ([#9085](https://github.com/opensearch-project/OpenSearch/pull/9085)) -- Add attributes to startSpan methods ([#9199](https://github.com/opensearch-project/OpenSearch/pull/9199)) -- [Refactor] Task foundation classes to core library - pt 1 ([#9082](https://github.com/opensearch-project/OpenSearch/pull/9082)) -- Add base class for parameterizing the search based tests #9083 ([#9083](https://github.com/opensearch-project/OpenSearch/pull/9083)) -- Add support for wrapping CollectorManager with profiling during concurrent execution ([#9129](https://github.com/opensearch-project/OpenSearch/pull/9129)) -- Rethrow OpenSearch exception for non-concurrent path while using concurrent search ([#9177](https://github.com/opensearch-project/OpenSearch/pull/9177)) -- Improve performance of encoding composite keys in multi-term aggregations ([#9412](https://github.com/opensearch-project/OpenSearch/pull/9412)) -- Fix sort related ITs for concurrent search ([#9177](https://github.com/opensearch-project/OpenSearch/pull/9466) -- Removing the vec file extension from INDEX_STORE_HYBRID_NIO_EXTENSIONS, to ensure the no performance degradation for vector search via Lucene Engine.([#9528](https://github.com/opensearch-project/OpenSearch/pull/9528))) -- Add support to use trace propagated from client ([#9506](https://github.com/opensearch-project/OpenSearch/pull/9506)) -- Separate request-based and settings-based concurrent segment search controls and introduce AggregatorFactory method to determine concurrent search support ([#9469](https://github.com/opensearch-project/OpenSearch/pull/9469)) -- [Remote Store] Rate limiter integration for remote store uploads and downloads([#9448](https://github.com/opensearch-project/OpenSearch/pull/9448/)) -- [Remote Store] Implicitly use replication type SEGMENT for remote store clusters ([#9264](https://github.com/opensearch-project/OpenSearch/pull/9264)) -- Redefine telemetry context restoration and propagation ([#9617](https://github.com/opensearch-project/OpenSearch/pull/9617)) -- Use non-concurrent path for sort request on timeseries index and field([#9562](https://github.com/opensearch-project/OpenSearch/pull/9562)) -- Added sampler based on `Blanket Probabilistic Sampling rate` and `Override for on demand` ([#9621](https://github.com/opensearch-project/OpenSearch/issues/9621)) -- Improve performance of rounding dates in date_histogram aggregation ([#9727](https://github.com/opensearch-project/OpenSearch/pull/9727)) -- [Remote Store] Add support for Remote Translog Store stats in `_remotestore/stats/` API ([#9263](https://github.com/opensearch-project/OpenSearch/pull/9263)) -- Add support for query profiler with concurrent aggregation ([#9248](https://github.com/opensearch-project/OpenSearch/pull/9248)) -- Cleanup Unreferenced file on segment merge failure ([#9503](https://github.com/opensearch-project/OpenSearch/pull/9503)) -- Move zstd compression codec to external custom-codecs repository ([#9422](https://github.com/opensearch-project/OpenSearch/issues/9422]) -- [Remote Store] Add support for Remote Translog Store upload stats in `_nodes/stats/` API ([#8908](https://github.com/opensearch-project/OpenSearch/pull/8908)) - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) - Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916)) - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) @@ -195,21 +86,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Deprecated ### Removed -- Remove provision to create Remote Indices without Remote Translog Store ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) ### Fixed -- Fix flaky ResourceAwareTasksTests.testBasicTaskResourceTracking test ([#8993](https://github.com/opensearch-project/OpenSearch/pull/8993)) -- Fix null_pointer_exception when creating or updating ingest pipeline ([#9259](https://github.com/opensearch-project/OpenSearch/pull/9259)) -- Fix memory leak when using Zstd Dictionary ([#9403](https://github.com/opensearch-project/OpenSearch/pull/9403)) -- Fix range reads in respository-s3 ([9512](https://github.com/opensearch-project/OpenSearch/issues/9512)) -- Handle null partSize in OnDemandBlockSnapshotIndexInput ([#9291](https://github.com/opensearch-project/OpenSearch/issues/9291)) -- Fix condition to remove index create block ([#9437](https://github.com/opensearch-project/OpenSearch/pull/9437)) -- Add support to clear archived index setting ([#9019](https://github.com/opensearch-project/OpenSearch/pull/9019)) -- [Segment Replication] Fixed bug where replica shard temporarily serves stale data during an engine reset ([#9495](https://github.com/opensearch-project/OpenSearch/pull/9495)) -- Disable shard/segment level search_after short cutting if track_total_hits != false ([#9683](https://github.com/opensearch-project/OpenSearch/pull/9683)) -- [Segment Replication] Fixed bug where bytes behind metric is not accurate ([#9686](https://github.com/opensearch-project/OpenSearch/pull/9686)) ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x diff --git a/release-notes/opensearch.release-notes-2.10.0.md b/release-notes/opensearch.release-notes-2.10.0.md new file mode 100644 index 0000000000000..9d5f75d61ee2a --- /dev/null +++ b/release-notes/opensearch.release-notes-2.10.0.md @@ -0,0 +1,136 @@ +## 2023-09-08 Version 2.10.0 Release Notes + +## [2.10] + +### Added +- Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583) +- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221) +- Introduce new static cluster setting to control slice computation for concurrent segment search. ([#8847](https://github.com/opensearch-project/OpenSearch/pull/8884)) +- Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606)) +- Disallow compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737) +- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195) +- Prioritize replica shard movement during shard relocation ([#8875](https://github.com/opensearch-project/OpenSearch/pull/8875)) +- Introducing Default and Best Compression codecs as their algorithm name ([#9123](https://github.com/opensearch-project/OpenSearch/pull/9123)) +- Make SearchTemplateRequest implement IndicesRequest.Replaceable ([#9122](https://github.com/opensearch-project/OpenSearch/pull/9122)) +- [BWC and API enforcement] Define the initial set of annotations, their meaning and relations between them ([#9223](https://github.com/opensearch-project/OpenSearch/pull/9223)) +- [Remote Store] Add Segment download stats to remotestore stats API ([#8718](https://github.com/opensearch-project/OpenSearch/pull/8718)) +- [Remote Store] Add remote segment transfer stats on NodesStats API ([#9168](https://github.com/opensearch-project/OpenSearch/pull/9168) [#9393](https://github.com/opensearch-project/OpenSearch/pull/9393) [#9454](https://github.com/opensearch-project/OpenSearch/pull/9454)) +- [Segment Replication] Support realtime reads for GET requests ([#9212](https://github.com/opensearch-project/OpenSearch/pull/9212)) +- Allow test clusters to run with TLS ([#8900](https://github.com/opensearch-project/OpenSearch/pull/8900)) +- Add jdk.incubator.vector module support for JDK 20+ ([#8601](https://github.com/opensearch-project/OpenSearch/pull/8601)) +- [Feature] Expose term frequency in Painless script score context ([#9081](https://github.com/opensearch-project/OpenSearch/pull/9081)) +- Add support for reading partial files to HDFS repository ([#9513](https://github.com/opensearch-project/OpenSearch/issues/9513)) +- [Remote Store] Rate limiter integration for remote store uploads and downloads([#9448](https://github.com/opensearch-project/OpenSearch/pull/9448/)) +- [BWC and API enforcement] Decorate the existing APIs with proper annotations (part 1) ([#9520](https://github.com/opensearch-project/OpenSearch/pull/9520)) +- Add support for extensions to search responses using SearchExtBuilder ([#9379](https://github.com/opensearch-project/OpenSearch/pull/9379)) +- [Remote State] Create service to publish cluster state to remote store ([#9160](https://github.com/opensearch-project/OpenSearch/pull/9160)) +- Core crypto library to perform encryption and decryption of source content ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466)) +- Expose DelimitedTermFrequencyTokenFilter to allow providing term frequencies along with terms ([#9479](https://github.com/opensearch-project/OpenSearch/pull/9479)) +- APIs for performing async blob reads and async downloads from the repository using multiple streams ([#9592](https://github.com/opensearch-project/OpenSearch/issues/9592)) +- Add concurrent segment search related metrics to node and index stats ([#9622](https://github.com/opensearch-project/OpenSearch/issues/9622)) +- Add average concurrency metric for concurrent segment search ([#9670](https://github.com/opensearch-project/OpenSearch/issues/9670)) +- Introduce cluster default remote translog buffer interval setting ([#9584](https://github.com/opensearch-project/OpenSearch/pull/9584)) +- Added encryption-sdk lib to provide encryption and decryption capabilities ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466) [#9289](https://github.com/opensearch-project/OpenSearch/pull/9289)) +- [Segment Replication] Adding segment replication statistics rolled up at index, node and cluster level ([#9709](https://github.com/opensearch-project/OpenSearch/pull/9709)) +- Added crypto-kms plugin to provide AWS KMS based key providers for encryption/decryption. ([#8465](https://github.com/opensearch-project/OpenSearch/pull/8465)) +- [Remote state] Integrate remote cluster state in publish/commit flow ([#9665](https://github.com/opensearch-project/OpenSearch/pull/9665)) +- [Remote Store] Changes to introduce repository registration during bootstrap via node attributes. ([#9105](https://github.com/opensearch-project/OpenSearch/pull/9105)) +- [Remote state] Auto restore index metadata from last known cluster state ([#9831](https://github.com/opensearch-project/OpenSearch/pull/9831)) + +### Dependencies +- Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) +- Bump `io.grpc:grpc-context` from 1.46.0 to 1.57.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726), [#9145](https://github.com/opensearch-project/OpenSearch/pull/9145)) +- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724)) +- Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) +- Bump `org.apache.zookeeper:zookeeper` from 3.8.1 to 3.9.0 ([#8844](https://github.com/opensearch-project/OpenSearch/pull/8844), [#9146](https://github.com/opensearch-project/OpenSearch/pull/9146)) +- Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842)) +- Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) +- Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) +- OpenJDK Update (July 2023 Patch releases) ([#8869](https://github.com/opensearch-project/OpenSearch/pull/8869)) +- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995)) +- Bump `com.gradle.enterprise` from 3.13.3 to 3.14.1 ([#8996](https://github.com/opensearch-project/OpenSearch/pull/8996)) +- Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 ([#8995](https://github.com/opensearch-project/OpenSearch/pull/8995)) +- Bump `com.google.cloud:google-cloud-core-http` from 2.21.0 to 2.21.1 ([#8999](https://github.com/opensearch-project/OpenSearch/pull/8999)) +- Bump `com.maxmind.geoip2:geoip2` from 4.0.1 to 4.1.0 ([#8998](https://github.com/opensearch-project/OpenSearch/pull/8998)) +- Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 in /plugins/repository-hdfs ([#8997](https://github.com/opensearch-project/OpenSearch/pull/8997)) +- Bump `netty` from 4.1.94.Final to 4.1.96.Final ([#9030](https://github.com/opensearch-project/OpenSearch/pull/9030)) +- Bump `com.google.jimfs:jimfs` from 1.2 to 1.3.0 ([#9080](https://github.com/opensearch-project/OpenSearch/pull/9080)) +- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.8 to 1.1.9 ([#9147](https://github.com/opensearch-project/OpenSearch/pull/9147)) +- Bump `org.apache.maven:maven-model` from 3.9.3 to 3.9.4 ([#9148](https://github.com/opensearch-project/OpenSearch/pull/9148)) +- Bump `com.azure:azure-storage-blob` from 12.22.3 to 12.23.0 ([#9231](https://github.com/opensearch-project/OpenSearch/pull/9231)) +- Bump `com.diffplug.spotless` from 6.19.0 to 6.20.0 ([#9227](https://github.com/opensearch-project/OpenSearch/pull/9227)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.8.2 to 1.1.10.3 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) +- Bump `com.squareup.okhttp3:okhttp` from 4.9.3 to 4.11.0 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) +- Bump `com.squareup.okio:okio` from 2.8.0 to 3.5.0 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) +- Bump `com.google.code.gson:gson` from 2.9.0 to 2.10.1 ([#9230](https://github.com/opensearch-project/OpenSearch/pull/9230)) +- Bump `lycheeverse/lychee-action` from 1.2.0 to 1.8.0 ([#9228](https://github.com/opensearch-project/OpenSearch/pull/9228)) +- Bump `snakeyaml` from 2.0 to 2.1 ([#9269](https://github.com/opensearch-project/OpenSearch/pull/9269)) +- Bump `aws-actions/configure-aws-credentials` from 1 to 2 ([#9302](https://github.com/opensearch-project/OpenSearch/pull/9302)) +- Bump `com.github.luben:zstd-jni` from 1.5.5-3 to 1.5.5-5 ([#9431](https://github.com/opensearch-project/OpenSearch/pull/9431) +- Bump `netty` from 4.1.96.Final to 4.1.97.Final ([#9553](https://github.com/opensearch-project/OpenSearch/pull/9553)) +- Bump `io.grpc:grpc-api` from 1.57.1 to 1.57.2 ([#9578](https://github.com/opensearch-project/OpenSearch/pull/9578)) +- Add Encryption SDK dependencies ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466)) + +### Changed +- Default to mmapfs within hybridfs ([#8508](https://github.com/opensearch-project/OpenSearch/pull/8508)) +- Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) +- Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) +- Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) +- [Refactor] StreamIO from common to core.common namespace in core lib ([#8157](https://github.com/opensearch-project/OpenSearch/pull/8157)) +- [Refactor] Remaining HPPC to java.util collections ([#8730](https://github.com/opensearch-project/OpenSearch/pull/8730)) +- Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) +- Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) +- Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805)) +- Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807)) +- Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792)) +- [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792)) +- Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) +- Performance improvements for BytesRefHash ([#8788](https://github.com/opensearch-project/OpenSearch/pull/8788)) +- Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801)) +- [Remove] Deprecated Fractional ByteSizeValue support #9005 ([#9005](https://github.com/opensearch-project/OpenSearch/pull/9005)) +- Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801)) +- [Remote Store] Restrict user override for remote store index level settings ([#8812](https://github.com/opensearch-project/OpenSearch/pull/8812)) +- [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636)) +- Make MultiBucketConsumerService thread safe to use across slices during search ([#9047](https://github.com/opensearch-project/OpenSearch/pull/9047)) +- Removed blocking wait in TransportGetSnapshotsAction which was exhausting generic threadpool ([#8377](https://github.com/opensearch-project/OpenSearch/pull/8377)) +- Adds support for tracing runnable scenarios ([#8831](https://github.com/opensearch-project/OpenSearch/pull/8831)) +- Change shard_size and shard_min_doc_count evaluation to happen in shard level reduce phase ([#9085](https://github.com/opensearch-project/OpenSearch/pull/9085)) +- Add attributes to startSpan methods ([#9199](https://github.com/opensearch-project/OpenSearch/pull/9199)) +- [Refactor] Task foundation classes to core library - pt 1 ([#9082](https://github.com/opensearch-project/OpenSearch/pull/9082)) +- Add support for wrapping CollectorManager with profiling during concurrent execution ([#9129](https://github.com/opensearch-project/OpenSearch/pull/9129)) +- Add base class for parameterizing the search based tests #9083 ([#9083](https://github.com/opensearch-project/OpenSearch/pull/9083)) +- Add support for wrapping CollectorManager with profiling during concurrent execution ([#9129](https://github.com/opensearch-project/OpenSearch/pull/9129)) +- Rethrow OpenSearch exception for non-concurrent path while using concurrent search ([#9177](https://github.com/opensearch-project/OpenSearch/pull/9177)) +- Improve performance of encoding composite keys in multi-term aggregations ([#9412](https://github.com/opensearch-project/OpenSearch/pull/9412)) +- Refactor Compressors from CompressorFactory to CompressorRegistry for extensibility ([#9262](https://github.com/opensearch-project/OpenSearch/pull/9262)) +- Fix sort related ITs for concurrent search ([#9177](https://github.com/opensearch-project/OpenSearch/pull/9466) +- [Remote Store] Implicitly use replication type SEGMENT for remote store clusters ([#9264](https://github.com/opensearch-project/OpenSearch/pull/9264)) +- Add support to use trace propagated from client ([#9506](https://github.com/opensearch-project/OpenSearch/pull/9506)) +- Separate request-based and settings-based concurrent segment search controls and introduce AggregatorFactory method to determine concurrent search support ([#9469](https://github.com/opensearch-project/OpenSearch/pull/9469)) +- [Remote Store] Rate limiter integration for remote store uploads and downloads([#9448](https://github.com/opensearch-project/OpenSearch/pull/9448/)) +- [Remote Store] Implicitly use replication type SEGMENT for remote store clusters ([#9264](https://github.com/opensearch-project/OpenSearch/pull/9264)) +- Redefine telemetry context restoration and propagation ([#9617](https://github.com/opensearch-project/OpenSearch/pull/9617)) +- Use non-concurrent path for sort request on timeseries index and field([#9562](https://github.com/opensearch-project/OpenSearch/pull/9562)) +- Added sampler based on `Blanket Probabilistic Sampling rate` and `Override for on demand` ([#9621](https://github.com/opensearch-project/OpenSearch/issues/9621)) +- Decouple replication lag from logic to fail stale replicas ([#9507](https://github.com/opensearch-project/OpenSearch/pull/9507)) +- Improve performance of rounding dates in date_histogram aggregation ([#9727](https://github.com/opensearch-project/OpenSearch/pull/9727)) +- [Remote Store] Add support for Remote Translog Store stats in `_remotestore/stats/` API ([#9263](https://github.com/opensearch-project/OpenSearch/pull/9263)) +- Removing the vec file extension from INDEX_STORE_HYBRID_NIO_EXTENSIONS, to ensure the no performance degradation for vector search via Lucene Engine.([#9528](https://github.com/opensearch-project/OpenSearch/pull/9528))) +- Cleanup Unreferenced file on segment merge failure ([#9503](https://github.com/opensearch-project/OpenSearch/pull/9503)) +- Move zstd compression codec to external custom-codecs repository ([#9422](https://github.com/opensearch-project/OpenSearch/issues/9422]) +- [Remote Store] Add support for Remote Translog Store upload stats in `_nodes/stats/` API ([#8908](https://github.com/opensearch-project/OpenSearch/pull/8908)) +- [Remote Store] Removing feature flag to mark feature GA ([#9761](https://github.com/opensearch-project/OpenSearch/pull/9761)) + +### Removed +- Remove provision to create Remote Indices without Remote Translog Store ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) + +### Fixed +- Fix flaky ResourceAwareTasksTests.testBasicTaskResourceTracking test ([#8993](https://github.com/opensearch-project/OpenSearch/pull/8993)) +- Fix null_pointer_exception when creating or updating ingest pipeline ([#9259](https://github.com/opensearch-project/OpenSearch/pull/9259)) +- Fix memory leak when using Zstd Dictionary ([#9403](https://github.com/opensearch-project/OpenSearch/pull/9403)) +- Fix condition to remove index create block ([#9437](https://github.com/opensearch-project/OpenSearch/pull/9437)) +- Add support to clear archived index setting ([#9019](https://github.com/opensearch-project/OpenSearch/pull/9019)) +- Fix range reads in respository-s3 ([9512](https://github.com/opensearch-project/OpenSearch/issues/9512)) +- [Segment Replication] Fixed bug where replica shard temporarily serves stale data during an engine reset ([#9495](https://github.com/opensearch-project/OpenSearch/pull/9495)) +- Disable shard/segment level search_after short cutting if track_total_hits != false ([#9683](https://github.com/opensearch-project/OpenSearch/pull/9683)) +- [Segment Replication] Fixed bug where bytes behind metric is not accurate ([#9686](https://github.com/opensearch-project/OpenSearch/pull/9686)) From c6fe7451d72c0f444553c05c0a46247886e941e2 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Fri, 8 Sep 2023 21:27:42 -0700 Subject: [PATCH 008/388] Mute CorrelationCodecTests.testCorrelationVectorIndex (#9956) Fix is pending in #7771, but that PR may take some time to land in main so muting for the time being. Signed-off-by: Andrew Ross --- .../core/index/codec/correlation950/CorrelationCodecTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java index b7b8ef0226d4e..b93172537d419 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java @@ -51,6 +51,7 @@ public class CorrelationCodecTests extends OpenSearchTestCase { * test correlation vector index * @throws Exception Exception */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8329") public void testCorrelationVectorIndex() throws Exception { Function perFieldCorrelationVectorsProvider = mapperService -> new PerFieldCorrelationVectorsFormat(Optional.of(mapperService)); From c100c0c8328dd792abd285d1d9b6aeed6619dd60 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Sat, 9 Sep 2023 08:40:24 -0700 Subject: [PATCH 009/388] Update single comment for check compatibility (#9954) Instead of appending a new comment every time check compatibility runs, this will instead update a single comment with the latest results. The history of previous runs is retained in the comment as GitHub allows you to view the edited revisions of any comment. This matches the behavior of the codecov workflow and should substantially reduce noise in PRs. Relates #9699 Signed-off-by: Andrew Ross --- .github/workflows/check-compatibility.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/check-compatibility.yml b/.github/workflows/check-compatibility.yml index dab8afd4ec1f2..4b8ef22351145 100644 --- a/.github/workflows/check-compatibility.yml +++ b/.github/workflows/check-compatibility.yml @@ -52,8 +52,18 @@ jobs: with: name: results.txt + - name: Find Comment + uses: peter-evans/find-comment@v2 + id: fc + with: + issue-number: ${{ github.event.number }} + comment-author: 'github-actions[bot]' + body-includes: 'Compatibility status:' + - name: Add comment on the PR uses: peter-evans/create-or-update-comment@v3 with: + comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} body-path: results.txt + edit-mode: replace From 77f0df31aa75aa855a8cac525569ae7faa4b9d1f Mon Sep 17 00:00:00 2001 From: Laurent Laborde Date: Sun, 10 Sep 2023 08:05:30 +0200 Subject: [PATCH 010/388] Fixing various javadocs (#9817) * fixing javadoc of org.opensearch.cli.Terminal Signed-off-by: Laurent Laborde * fixing javadoc of org.opensearch.cli.ExitCodes Signed-off-by: Laurent Laborde * fixing javadoc Signed-off-by: Laurent Laborde * fixing a mishap in formatting rule Signed-off-by: Laurent Laborde --------- Signed-off-by: Laurent Laborde --- .../java/org/opensearch/cli/ExitCodes.java | 40 ++++--- .../java/org/opensearch/cli/Terminal.java | 103 ++++++++++++++---- .../org/opensearch/common/collect/Tuple.java | 22 ++++ .../common/crypto/CryptoHandler.java | 5 +- .../opensearch/common/crypto/DataKeyPair.java | 5 +- .../crypto/DecryptedRangedStreamProvider.java | 2 + .../common/crypto/MasterKeyProvider.java | 3 +- .../compress/spi/CompressionProvider.java | 5 +- 8 files changed, 147 insertions(+), 38 deletions(-) diff --git a/libs/cli/src/main/java/org/opensearch/cli/ExitCodes.java b/libs/cli/src/main/java/org/opensearch/cli/ExitCodes.java index c705177b0d7b6..90efc89a08caf 100644 --- a/libs/cli/src/main/java/org/opensearch/cli/ExitCodes.java +++ b/libs/cli/src/main/java/org/opensearch/cli/ExitCodes.java @@ -36,20 +36,34 @@ * POSIX exit codes. */ public class ExitCodes { + /** No error */ public static final int OK = 0; - public static final int USAGE = 64; /* command line usage error */ - public static final int DATA_ERROR = 65; /* data format error */ - public static final int NO_INPUT = 66; /* cannot open input */ - public static final int NO_USER = 67; /* addressee unknown */ - public static final int NO_HOST = 68; /* host name unknown */ - public static final int UNAVAILABLE = 69; /* service unavailable */ - public static final int CODE_ERROR = 70; /* internal software error */ - public static final int CANT_CREATE = 73; /* can't create (user) output file */ - public static final int IO_ERROR = 74; /* input/output error */ - public static final int TEMP_FAILURE = 75; /* temp failure; user is invited to retry */ - public static final int PROTOCOL = 76; /* remote error in protocol */ - public static final int NOPERM = 77; /* permission denied */ - public static final int CONFIG = 78; /* configuration error */ + /** command line usage error */ + public static final int USAGE = 64; + /** data format error */ + public static final int DATA_ERROR = 65; + /** cannot open input */ + public static final int NO_INPUT = 66; + /** addressee unknown */ + public static final int NO_USER = 67; + /** host name unknown */ + public static final int NO_HOST = 68; + /** service unavailable */ + public static final int UNAVAILABLE = 69; + /** internal software error */ + public static final int CODE_ERROR = 70; + /** can't create (user) output file */ + public static final int CANT_CREATE = 73; + /** input/output error */ + public static final int IO_ERROR = 74; + /** temp failure; user is invited to retry */ + public static final int TEMP_FAILURE = 75; + /** remote error in protocol */ + public static final int PROTOCOL = 76; + /** permission denied */ + public static final int NOPERM = 77; + /** configuration error */ + public static final int CONFIG = 78; private ExitCodes() { /* no instance, just constants */ } } diff --git a/libs/cli/src/main/java/org/opensearch/cli/Terminal.java b/libs/cli/src/main/java/org/opensearch/cli/Terminal.java index 657b95fa052ab..be030c18507ad 100644 --- a/libs/cli/src/main/java/org/opensearch/cli/Terminal.java +++ b/libs/cli/src/main/java/org/opensearch/cli/Terminal.java @@ -51,6 +51,8 @@ * which allows {@link #println(Verbosity,String)} calls which act like a logger, * only actually printing if the verbosity level of the terminal is above * the verbosity of the message. + * @see ConsoleTerminal + * @see SystemTerminal */ public abstract class Terminal { @@ -65,35 +67,57 @@ private static PrintWriter newErrorWriter() { return new PrintWriter(System.err); } - /** Defines the available verbosity levels of messages to be printed. */ + /** Defines the available verbosity levels of messages to be printed.*/ public enum Verbosity { - SILENT, /* always printed */ - NORMAL, /* printed when no options are given to cli */ - VERBOSE /* printed only when cli is passed verbose option */ + /** always printed */ + SILENT, + /** printed when no options are given to cli */ + NORMAL, + /** printed only when cli is passed verbose option */ + VERBOSE } /** The current verbosity for the terminal, defaulting to {@link Verbosity#NORMAL}. */ private Verbosity verbosity = Verbosity.NORMAL; - /** The newline used when calling println. */ + /** The newline separator used when calling println. */ private final String lineSeparator; + /** Constructs a new terminal with the given line separator. + * @param lineSeparator the line separator to use when calling println + * */ protected Terminal(String lineSeparator) { this.lineSeparator = lineSeparator; } - /** Sets the verbosity of the terminal. */ + /** Sets the {@link Terminal#verbosity} of the terminal. (Default is {@link Verbosity#NORMAL}) + * @param verbosity the {@link Verbosity} level that will be used for printing + * */ public void setVerbosity(Verbosity verbosity) { this.verbosity = verbosity; } - /** Reads clear text from the terminal input. See {@link Console#readLine()}. */ + /** Reads clear text from the terminal input. + * @see Console#readLine() + * @param prompt message to display to the user + * @return the text entered by the user + * */ public abstract String readText(String prompt); - /** Reads password text from the terminal input. See {@link Console#readPassword()}}. */ + /** Reads secret text from the terminal input with echoing disabled. + * @see Console#readPassword() + * @param prompt message to display to the user + * @return the secret as a character array + * */ public abstract char[] readSecret(String prompt); - /** Read password text form terminal input up to a maximum length. */ + /** Read secret text from terminal input with echoing disabled, up to a maximum length. + * @see Console#readPassword() + * @param prompt message to display to the user + * @param maxLength the maximum length of the secret + * @return the secret as a character array + * @throws IllegalStateException if the secret exceeds the maximum length + * */ public char[] readSecret(String prompt, int maxLength) { char[] result = readSecret(prompt); if (result.length > maxLength) { @@ -103,30 +127,48 @@ public char[] readSecret(String prompt, int maxLength) { return result; } - /** Returns a Writer which can be used to write to the terminal directly using standard output. */ + /** Returns a Writer which can be used to write to the terminal directly using standard output. + * @return a writer to {@link Terminal#DEFAULT} output + * @see Terminal.ConsoleTerminal + * @see Terminal.SystemTerminal + * */ public abstract PrintWriter getWriter(); - /** Returns a Writer which can be used to write to the terminal directly using standard error. */ + /** Returns a Writer which can be used to write to the terminal directly using standard error. + * @return a writer to stderr + * */ public PrintWriter getErrorWriter() { return ERROR_WRITER; } - /** Prints a line to the terminal at {@link Verbosity#NORMAL} verbosity level. */ + /** Prints a line to the terminal at {@link Verbosity#NORMAL} verbosity level, with a {@link Terminal#lineSeparator} + * @param msg the message to print + * */ public final void println(String msg) { println(Verbosity.NORMAL, msg); } - /** Prints a line to the terminal at {@code verbosity} level. */ + /** Prints message to the terminal's standard output at {@link Verbosity} level, with a {@link Terminal#lineSeparator}. + * @param verbosity the {@link Verbosity} level at which to print + * @param msg the message to print + * */ public final void println(Verbosity verbosity, String msg) { print(verbosity, msg + lineSeparator); } - /** Prints message to the terminal's standard output at {@code verbosity} level, without a newline. */ + /** Prints message to the terminal's standard output at {@link Verbosity} level, without adding a {@link Terminal#lineSeparator}. + * @param verbosity the {@link Verbosity} level at which to print + * @param msg the message to print + * */ public final void print(Verbosity verbosity, String msg) { print(verbosity, msg, false); } - /** Prints message to the terminal at {@code verbosity} level, without a newline. */ + /** Prints message to either standard or error output at {@link Verbosity} level, without adding a {@link Terminal#lineSeparator}. + * @param verbosity the {@link Verbosity} level at which to print. + * @param msg the message to print + * @param isError if true, prints to standard error instead of standard output + * */ private void print(Verbosity verbosity, String msg, boolean isError) { if (isPrintable(verbosity)) { PrintWriter writer = isError ? getErrorWriter() : getWriter(); @@ -135,29 +177,44 @@ private void print(Verbosity verbosity, String msg, boolean isError) { } } - /** Prints a line to the terminal's standard error at {@link Verbosity#NORMAL} verbosity level, without a newline. */ + /** Prints a line to the terminal's standard error at {@link Verbosity} level, without adding a {@link Terminal#lineSeparator}. + * @param verbosity the {@link Verbosity} level at which to print. + * @param msg the message to print + * */ public final void errorPrint(Verbosity verbosity, String msg) { print(verbosity, msg, true); } - /** Prints a line to the terminal's standard error at {@link Verbosity#NORMAL} verbosity level. */ + /** Prints a line to the terminal's standard error at {@link Verbosity#NORMAL} verbosity level, with a {@link Terminal#lineSeparator} + * @param msg the message to print + * */ public final void errorPrintln(String msg) { errorPrintln(Verbosity.NORMAL, msg); } - /** Prints a line to the terminal's standard error at {@code verbosity} level. */ + /** Prints a line to the terminal's standard error at {@link Verbosity} level, with a {@link Terminal#lineSeparator}. + * @param verbosity the {@link Verbosity} level at which to print. + * @param msg the message to print + * */ public final void errorPrintln(Verbosity verbosity, String msg) { errorPrint(verbosity, msg + lineSeparator); } - /** Checks if is enough {@code verbosity} level to be printed */ + /** Checks if given {@link Verbosity} level is high enough to be printed at the level defined by {@link Terminal#verbosity} + * @param verbosity the {@link Verbosity} level to check + * @return true if the {@link Verbosity} level is high enough to be printed + * @see Terminal#setVerbosity(Verbosity) + * */ public final boolean isPrintable(Verbosity verbosity) { return this.verbosity.ordinal() >= verbosity.ordinal(); } /** - * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n' + * Prompt for a yes or no answer from the user. This method will loop until 'y', 'n' * (or the default empty value) is entered. + * @param prompt the prompt to display to the user + * @param defaultYes if true, the default answer is 'y', otherwise it is 'n' + * @return true if the user answered 'y', false if the user answered 'n' or the defaultYes value if the user entered nothing */ public final boolean promptYesNo(String prompt, boolean defaultYes) { String answerPrompt = defaultYes ? " [Y/n]" : " [y/N]"; @@ -181,6 +238,11 @@ public final boolean promptYesNo(String prompt, boolean defaultYes) { * character is immediately preceded by a carriage return, we have * a Windows-style newline, so we discard the carriage return as well * as the newline. + * @param reader the reader to read from + * @param maxLength the maximum length of the line to read + * @return the line read from the reader + * @throws RuntimeException if the line read exceeds the maximum length + * @throws RuntimeException if an IOException occurs while reading */ public static char[] readLineToCharArray(Reader reader, int maxLength) { char[] buf = new char[maxLength + 2]; @@ -215,6 +277,7 @@ public static char[] readLineToCharArray(Reader reader, int maxLength) { } } + /** Flushes the terminal's standard output and standard error. */ public void flush() { this.getWriter().flush(); this.getErrorWriter().flush(); diff --git a/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java b/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java index 36bc5504061f5..d0b94536b0729 100644 --- a/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java +++ b/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java @@ -61,6 +61,20 @@ public V2 v2() { return v2; } + /** + * Returns {@code true} if the given object is also a tuple and the two tuples + * have equal {@link #v1()} and {@link #v2()} values. + *

+ * Returns {@code false} otherwise, including for {@code null} values or + * objects of different types. + *

+ * Note: {@code Tuple} instances are equal if the underlying values are + * equal, even if the types are different. + * + * @param o the object to compare to + * @return {@code true} if the given object is also a tuple and the two tuples + * have equal {@link #v1()} and {@link #v2()} values. + */ @Override public boolean equals(Object o) { if (this == o) return true; @@ -74,6 +88,10 @@ public boolean equals(Object o) { return true; } + /** + * Returns the hash code value for this Tuple. + * @return the hash code value for this Tuple. + */ @Override public int hashCode() { int result = v1 != null ? v1.hashCode() : 0; @@ -81,6 +99,10 @@ public int hashCode() { return result; } + /** + * Returns a string representation of a Tuple + * @return {@code "Tuple [v1=value1, v2=value2]"} + */ @Override public String toString() { return "Tuple [v1=" + v1 + ", v2=" + v2 + "]"; diff --git a/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java b/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java index 37322310c7287..9572b5b9054b2 100644 --- a/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java +++ b/libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java @@ -19,8 +19,8 @@ * Crypto provider abstractions for encryption and decryption of data. Allows registering multiple providers * for defining different ways of encrypting or decrypting data. * - * T - Encryption Metadata / CryptoContext - * U - Parsed Encryption Metadata / CryptoContext + * @param Encryption Metadata / CryptoContext + * @param Parsed Encryption Metadata / CryptoContext */ @ExperimentalApi public interface CryptoHandler extends Closeable { @@ -38,6 +38,7 @@ public interface CryptoHandler extends Closeable { * Note that underlying information in the loaded metadata object is same as present in the object created during * encryption but object type may differ. * + * @param encryptedHeaderContentSupplier supplier for encrypted header content. * @return crypto metadata instance used in decryption. */ U loadEncryptionMetadata(EncryptedHeaderContentSupplier encryptedHeaderContentSupplier) throws IOException; diff --git a/libs/common/src/main/java/org/opensearch/common/crypto/DataKeyPair.java b/libs/common/src/main/java/org/opensearch/common/crypto/DataKeyPair.java index 3d487f9028d71..711c0d314ecef 100644 --- a/libs/common/src/main/java/org/opensearch/common/crypto/DataKeyPair.java +++ b/libs/common/src/main/java/org/opensearch/common/crypto/DataKeyPair.java @@ -11,7 +11,10 @@ * Key pair generated by {@link MasterKeyProvider} */ public class DataKeyPair { + + /** Unencrypted data key used for encryption and decryption */ private final byte[] rawKey; + /** Encrypted version of rawKey */ private final byte[] encryptedKey; /** @@ -25,7 +28,7 @@ public DataKeyPair(byte[] rawKey, byte[] encryptedKey) { } /** - * Returns raw key + * Returns Unencrypted data key * @return raw/decrypted key */ public byte[] getRawKey() { diff --git a/libs/common/src/main/java/org/opensearch/common/crypto/DecryptedRangedStreamProvider.java b/libs/common/src/main/java/org/opensearch/common/crypto/DecryptedRangedStreamProvider.java index 06b18027a0726..2cda3c1f8bdb4 100644 --- a/libs/common/src/main/java/org/opensearch/common/crypto/DecryptedRangedStreamProvider.java +++ b/libs/common/src/main/java/org/opensearch/common/crypto/DecryptedRangedStreamProvider.java @@ -16,7 +16,9 @@ */ public class DecryptedRangedStreamProvider { + /** Adjusted range of partial encrypted content which needs to be used for decryption. */ private final long[] adjustedRange; + /** Stream provider for decryption and range re-adjustment. */ private final UnaryOperator decryptedStreamProvider; /** diff --git a/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java b/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java index b5feeb6d37ec6..8afa48eb92c0f 100644 --- a/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java +++ b/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java @@ -22,7 +22,7 @@ public interface MasterKeyProvider extends Closeable { DataKeyPair generateDataPair(); /** - * Returns decrpted key against the encrypted key. + * Returns decrypted key against the encrypted key. * @param encryptedKey Key to decrypt * @return Decrypted version of key. */ @@ -35,6 +35,7 @@ public interface MasterKeyProvider extends Closeable { String getKeyId(); /** + * Returns encryption context associated with this master key. * @return encryption context associated with this master key. */ Map getEncryptionContext(); diff --git a/libs/compress/src/main/java/org/opensearch/compress/spi/CompressionProvider.java b/libs/compress/src/main/java/org/opensearch/compress/spi/CompressionProvider.java index 58bf24a210bae..f0c6969377d78 100644 --- a/libs/compress/src/main/java/org/opensearch/compress/spi/CompressionProvider.java +++ b/libs/compress/src/main/java/org/opensearch/compress/spi/CompressionProvider.java @@ -23,7 +23,10 @@ */ public class CompressionProvider implements CompressorProvider { - /** Returns the concrete {@link Compressor}s provided by the compress library */ + /** + * Returns the concrete {@link Compressor}s provided by the compress library + * @return a list of {@link Compressor}s + * */ @SuppressWarnings({ "unchecked", "rawtypes" }) @Override public List> getCompressors() { From 1c8d1716bc41c94a5f79a5b9838856453fe36d75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 Sep 2023 10:35:09 -0400 Subject: [PATCH 011/388] Bump peter-evans/create-or-update-comment from 2 to 3 (#9575) * Bump peter-evans/create-or-update-comment from 2 to 3 Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 2 to 3. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/v2...v3) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/gradle-check.yml | 6 +++--- .github/workflows/poc-checklist.yml | 2 +- CHANGELOG.md | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index b86962a5ebb49..67abb5bcb8ffe 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -78,7 +78,7 @@ jobs: - name: Create Comment Success if: ${{ github.event_name == 'pull_request_target' && success() && env.result == 'SUCCESS' }} - uses: peter-evans/create-or-update-comment@v2 + uses: peter-evans/create-or-update-comment@v3 with: issue-number: ${{ env.pr_number }} body: | @@ -104,7 +104,7 @@ jobs: - name: Create Comment Flaky if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }} - uses: peter-evans/create-or-update-comment@v2 + uses: peter-evans/create-or-update-comment@v3 with: issue-number: ${{ env.pr_number }} body: | @@ -116,7 +116,7 @@ jobs: - name: Create Comment Failure if: ${{ github.event_name == 'pull_request_target' && failure() }} - uses: peter-evans/create-or-update-comment@v2 + uses: peter-evans/create-or-update-comment@v3 with: issue-number: ${{ env.pr_number }} body: | diff --git a/.github/workflows/poc-checklist.yml b/.github/workflows/poc-checklist.yml index 2dfb1bbe5cdce..3d014e000a487 100644 --- a/.github/workflows/poc-checklist.yml +++ b/.github/workflows/poc-checklist.yml @@ -11,7 +11,7 @@ jobs: issues: write steps: - name: Add comment - uses: peter-evans/create-or-update-comment@v2 + uses: peter-evans/create-or-update-comment@v3 with: issue-number: ${{ github.event.issue.number }} body: | diff --git a/CHANGELOG.md b/CHANGELOG.md index ec6ea2f79f22c..685daf7cf46fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added ### Dependencies +- Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) @@ -92,4 +93,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file From b7dbf46e9ec1d5c8cea4a95266f325a5a7520249 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:27:08 -0400 Subject: [PATCH 012/388] Bump actions/checkout from 2 to 4 (#9968) * Bump actions/checkout from 2 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/auto-release.yml | 2 +- .github/workflows/changelog_verifier.yml | 2 +- .github/workflows/check-compatibility.yml | 2 +- .github/workflows/create-documentation-issue.yml | 2 +- .github/workflows/dependabot_pr.yml | 2 +- .github/workflows/gradle-check.yml | 4 ++-- .github/workflows/links.yml | 2 +- .github/workflows/lucene-snapshots.yml | 4 ++-- .github/workflows/precommit.yml | 2 +- .github/workflows/publish-maven-snapshots.yml | 2 +- .github/workflows/version.yml | 8 ++++---- .github/workflows/wrapper.yml | 2 +- CHANGELOG.md | 1 + 13 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index b8d3912c5864a..ae41ae00d018d 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -22,7 +22,7 @@ jobs: - name: Get tag id: tag uses: dawidd6/action-get-tag@v1 - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ncipollo/release-action@v1 with: github_token: ${{ steps.github_app_token.outputs.token }} diff --git a/.github/workflows/changelog_verifier.yml b/.github/workflows/changelog_verifier.yml index 8060ea93f477a..9456fbf8b4ca0 100644 --- a/.github/workflows/changelog_verifier.yml +++ b/.github/workflows/changelog_verifier.yml @@ -9,7 +9,7 @@ jobs: if: github.repository == 'opensearch-project/OpenSearch' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/check-compatibility.yml b/.github/workflows/check-compatibility.yml index 4b8ef22351145..d93f7e73b91e7 100644 --- a/.github/workflows/check-compatibility.yml +++ b/.github/workflows/check-compatibility.yml @@ -11,7 +11,7 @@ jobs: contents: read runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/create-documentation-issue.yml b/.github/workflows/create-documentation-issue.yml index c81f7355a0d22..373eeb3569a51 100644 --- a/.github/workflows/create-documentation-issue.yml +++ b/.github/workflows/create-documentation-issue.yml @@ -21,7 +21,7 @@ jobs: installation_id: 22958780 - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Edit the issue template run: | diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml index 6f5058d9df54c..f03a3d125067d 100644 --- a/.github/workflows/dependabot_pr.yml +++ b/.github/workflows/dependabot_pr.yml @@ -18,7 +18,7 @@ jobs: installation_id: 22958780 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: token: ${{ steps.github_app_token.outputs.token }} diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index 67abb5bcb8ffe..bda3bbbc1f125 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -23,7 +23,7 @@ jobs: timeout-minutes: 130 steps: - name: Checkout OpenSearch repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} @@ -50,7 +50,7 @@ jobs: echo "pr_number=Null" >> $GITHUB_ENV - name: Checkout opensearch-build repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: opensearch-project/opensearch-build ref: main diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 4322a740145e6..ca026f530b4af 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: lychee Link Checker id: lychee uses: lycheeverse/lychee-action@v1.8.0 diff --git a/.github/workflows/lucene-snapshots.yml b/.github/workflows/lucene-snapshots.yml index 994b420cb5847..c2a2cedaaefb4 100644 --- a/.github/workflows/lucene-snapshots.yml +++ b/.github/workflows/lucene-snapshots.yml @@ -21,7 +21,7 @@ jobs: contents: read steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK 17 uses: actions/setup-java@v3 with: @@ -29,7 +29,7 @@ jobs: distribution: 'adopt' - name: Checkout Lucene - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: 'apache/lucene' path: lucene diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 0372d57dda91f..f562b46d5b124 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -9,7 +9,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml index 43c18af78ae4c..8c08df269a999 100644 --- a/.github/workflows/publish-maven-snapshots.yml +++ b/.github/workflows/publish-maven-snapshots.yml @@ -18,7 +18,7 @@ jobs: contents: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up JDK 17 uses: actions/setup-java@v3 with: diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index ad79a425557bb..2fe4fb1ac9d48 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -19,7 +19,7 @@ jobs: private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Fetch Tag and Version Information run: | TAG=$(echo "${GITHUB_REF#refs/*/}") @@ -44,7 +44,7 @@ jobs: echo "NEXT_VERSION=$NEXT_VERSION" >> $GITHUB_ENV echo "NEXT_VERSION_UNDERSCORE=$NEXT_VERSION_UNDERSCORE" >> $GITHUB_ENV echo "NEXT_VERSION_ID=$NEXT_VERSION_ID" >> $GITHUB_ENV - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: ${{ env.BASE }} token: ${{ steps.github_app_token.outputs.token }} @@ -73,7 +73,7 @@ jobs: body: | I've noticed that a new tag ${{ env.TAG }} was pushed, and incremented the version from ${{ env.CURRENT_VERSION }} to ${{ env.NEXT_VERSION }}. - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: ${{ env.BASE_X }} token: ${{ steps.github_app_token.outputs.token }} @@ -100,7 +100,7 @@ jobs: body: | I've noticed that a new tag ${{ env.TAG }} was pushed, and added a bwc version ${{ env.NEXT_VERSION }}. - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: main token: ${{ steps.github_app_token.outputs.token }} diff --git a/.github/workflows/wrapper.yml b/.github/workflows/wrapper.yml index 80acaa906711b..6dd48ca15eaa9 100644 --- a/.github/workflows/wrapper.yml +++ b/.github/workflows/wrapper.yml @@ -7,5 +7,5 @@ jobs: if: github.repository == 'opensearch-project/OpenSearch' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 685daf7cf46fd..1edd7bb20de2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) +- Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) From 05544ae2b9d8a7d4cd6dc0241dc31126a154fa4d Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 11 Sep 2023 13:58:11 -0400 Subject: [PATCH 013/388] Bump org.apache.commons:commons-compress from 1.23.0 to 1.24.0 (#9973) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 1 + distribution/tools/plugin-cli/build.gradle | 2 +- .../tools/plugin-cli/licenses/commons-compress-1.23.0.jar.sha1 | 1 - .../tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1 | 1 + plugins/ingest-attachment/build.gradle | 2 +- .../ingest-attachment/licenses/commons-compress-1.23.0.jar.sha1 | 1 - .../ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1 | 1 + plugins/repository-hdfs/build.gradle | 2 +- .../repository-hdfs/licenses/commons-compress-1.23.0.jar.sha1 | 1 - .../repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1 | 1 + 11 files changed, 8 insertions(+), 6 deletions(-) delete mode 100644 distribution/tools/plugin-cli/licenses/commons-compress-1.23.0.jar.sha1 create mode 100644 distribution/tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.23.0.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/commons-compress-1.23.0.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1edd7bb20de2e..f6d25133d3be5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) - Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968)) +- Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973](https://github.com/opensearch-project/OpenSearch/pull/9973)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 500727e909494..b658d45b815c5 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -40,6 +40,7 @@ httpasyncclient = 4.1.5 commonslogging = 1.2 commonscodec = 1.15 commonslang = 3.13.0 +commonscompress = 1.24.0 # plugin dependencies aws = 2.20.55 reactivestreams = 1.0.4 diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 5103999428814..2db3fef55d02e 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -45,7 +45,7 @@ dependencies { transitive = false } - implementation 'org.apache.commons:commons-compress:1.23.0' + implementation "org.apache.commons:commons-compress:${versions.commonscompress}" } tasks.named("dependencyLicenses").configure { diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.23.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.23.0.jar.sha1 deleted file mode 100644 index 48dba88409c17..0000000000000 --- a/distribution/tools/plugin-cli/licenses/commons-compress-1.23.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4af2060ea9b0c8b74f1854c6cafe4d43cfc161fc \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1 new file mode 100644 index 0000000000000..23999d1bfbde4 --- /dev/null +++ b/distribution/tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1 @@ -0,0 +1 @@ +b4b1b5a3d9573b2970fddab236102c0a4d27d35e \ No newline at end of file diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 2fe5b93704585..f54c80e9f76c1 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -84,7 +84,7 @@ dependencies { // MS Office api "org.apache.poi:poi-scratchpad:${versions.poi}" // Apple iWork - api 'org.apache.commons:commons-compress:1.23.0' + api "org.apache.commons:commons-compress:${versions.commonscompress}" // Outlook documents api "org.apache.james:apache-mime4j-core:${versions.mime4j}" api "org.apache.james:apache-mime4j-dom:${versions.mime4j}" diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.23.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.23.0.jar.sha1 deleted file mode 100644 index 48dba88409c17..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-compress-1.23.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4af2060ea9b0c8b74f1854c6cafe4d43cfc161fc \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1 new file mode 100644 index 0000000000000..23999d1bfbde4 --- /dev/null +++ b/plugins/ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1 @@ -0,0 +1 @@ +b4b1b5a3d9573b2970fddab236102c0a4d27d35e \ No newline at end of file diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 1fdb3d2fb41e2..e2f5707ca4393 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -73,7 +73,7 @@ dependencies { api 'commons-cli:commons-cli:1.5.0' api "commons-codec:commons-codec:${versions.commonscodec}" api 'commons-collections:commons-collections:3.2.2' - api 'org.apache.commons:commons-compress:1.23.0' + api "org.apache.commons:commons-compress:${versions.commonscompress}" api 'org.apache.commons:commons-configuration2:2.9.0' api 'commons-io:commons-io:2.13.0' api 'org.apache.commons:commons-lang3:3.13.0' diff --git a/plugins/repository-hdfs/licenses/commons-compress-1.23.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-compress-1.23.0.jar.sha1 deleted file mode 100644 index 48dba88409c17..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-compress-1.23.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4af2060ea9b0c8b74f1854c6cafe4d43cfc161fc \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1 new file mode 100644 index 0000000000000..23999d1bfbde4 --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1 @@ -0,0 +1 @@ +b4b1b5a3d9573b2970fddab236102c0a4d27d35e \ No newline at end of file From 04f90dc3a1e859cc2caf87a223006c188a1252f4 Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Mon, 11 Sep 2023 11:40:51 -0700 Subject: [PATCH 014/388] Bump JNA version from 5.5 to 5.13. (#9963) Update changelog. Signed-off-by: Austin Lee --- CHANGELOG.md | 3 ++- buildSrc/version.properties | 2 +- server/licenses/jna-5.13.0.jar.sha1 | 1 + server/licenses/jna-5.5.0.jar.sha1 | 1 - 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 server/licenses/jna-5.13.0.jar.sha1 delete mode 100644 server/licenses/jna-5.5.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d25133d3be5..cad0c2953fcdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) +- Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) @@ -95,4 +96,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x diff --git a/buildSrc/version.properties b/buildSrc/version.properties index b658d45b815c5..66679bad5c597 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -26,7 +26,7 @@ protobuf = 3.22.3 jakarta_annotation = 1.3.5 # when updating the JNA version, also update the version in buildSrc/build.gradle -jna = 5.5.0 +jna = 5.13.0 netty = 4.1.97.Final joda = 2.12.2 diff --git a/server/licenses/jna-5.13.0.jar.sha1 b/server/licenses/jna-5.13.0.jar.sha1 new file mode 100644 index 0000000000000..faf2012f0b5c0 --- /dev/null +++ b/server/licenses/jna-5.13.0.jar.sha1 @@ -0,0 +1 @@ +1200e7ebeedbe0d10062093f32925a912020e747 \ No newline at end of file diff --git a/server/licenses/jna-5.5.0.jar.sha1 b/server/licenses/jna-5.5.0.jar.sha1 deleted file mode 100644 index 5621dfc743dd0..0000000000000 --- a/server/licenses/jna-5.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0e0845217c4907822403912ad6828d8e0b256208 From b90a8887c29645aebc6de37325e369e0792aa9b7 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 11 Sep 2023 15:13:24 -0400 Subject: [PATCH 015/388] Bump OpenTelemetry from 1.26.0 to 1.30.0 (#9950) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 3 +- plugins/telemetry-otel/build.gradle | 22 +- .../opentelemetry-api-1.26.0.jar.sha1 | 1 - .../opentelemetry-api-1.30.1.jar.sha1 | 1 + ...entelemetry-api-logs-1.26.0-alpha.jar.sha1 | 1 - .../opentelemetry-api-logs-LICENSE.txt | 202 ------------------ .../opentelemetry-api-logs-NOTICE.txt | 0 .../opentelemetry-context-1.26.0.jar.sha1 | 1 - .../opentelemetry-context-1.30.1.jar.sha1 | 1 + ...telemetry-exporter-logging-1.26.0.jar.sha1 | 1 - ...telemetry-exporter-logging-1.30.1.jar.sha1 | 1 + .../opentelemetry-sdk-1.26.0.jar.sha1 | 1 - .../opentelemetry-sdk-1.30.1.jar.sha1 | 1 + .../opentelemetry-sdk-common-1.26.0.jar.sha1 | 1 - .../opentelemetry-sdk-common-1.30.1.jar.sha1 | 1 + ...entelemetry-sdk-logs-1.26.0-alpha.jar.sha1 | 1 - .../opentelemetry-sdk-logs-1.30.1.jar.sha1 | 1 + .../opentelemetry-sdk-metrics-1.26.0.jar.sha1 | 1 - .../opentelemetry-sdk-metrics-1.30.1.jar.sha1 | 1 + .../opentelemetry-sdk-trace-1.26.0.jar.sha1 | 1 - .../opentelemetry-sdk-trace-1.30.1.jar.sha1 | 1 + ...pentelemetry-semconv-1.26.0-alpha.jar.sha1 | 1 - ...pentelemetry-semconv-1.30.1-alpha.jar.sha1 | 1 + 24 files changed, 29 insertions(+), 218 deletions(-) delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-logs-NOTICE.txt delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index cad0c2953fcdd..d3e6de03709f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) - Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968)) +- Bump OpenTelemetry from 1.26.0 to 1.30.1 ([#9950](https://github.com/opensearch-project/OpenSearch/pull/9950)) - Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973](https://github.com/opensearch-project/OpenSearch/pull/9973)) ### Changed diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 66679bad5c597..ee666f90f83dd 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -68,5 +68,4 @@ jzlib = 1.1.3 resteasy = 6.2.4.Final # opentelemetry dependencies -opentelemetry = 1.26.0 - +opentelemetry = 1.30.1 diff --git a/plugins/telemetry-otel/build.gradle b/plugins/telemetry-otel/build.gradle index 04fa9df9a47d0..511bae590bf13 100644 --- a/plugins/telemetry-otel/build.gradle +++ b/plugins/telemetry-otel/build.gradle @@ -29,8 +29,7 @@ dependencies { api "io.opentelemetry:opentelemetry-sdk-metrics:${versions.opentelemetry}" api "io.opentelemetry:opentelemetry-exporter-logging:${versions.opentelemetry}" api "io.opentelemetry:opentelemetry-semconv:${versions.opentelemetry}-alpha" - api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}-alpha" - api "io.opentelemetry:opentelemetry-api-logs:${versions.opentelemetry}-alpha" + api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}" testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${versions.opentelemetry}" } @@ -50,11 +49,26 @@ thirdPartyAudit { 'io.opentelemetry.api.events.EventEmitterProvider', 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleHistogramBuilder', 'io.opentelemetry.extension.incubator.metrics.ExtendedLongHistogramBuilder', - 'io.opentelemetry.extension.incubator.metrics.HistogramAdviceConfigurer', 'io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties', 'io.opentelemetry.sdk.autoconfigure.spi.logs.ConfigurableLogRecordExporterProvider', 'io.opentelemetry.sdk.autoconfigure.spi.metrics.ConfigurableMetricExporterProvider', - 'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider' + 'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider', + 'io.opentelemetry.extension.incubator.metrics.DoubleCounterAdviceConfigurer', + 'io.opentelemetry.extension.incubator.metrics.DoubleGauge', + 'io.opentelemetry.extension.incubator.metrics.DoubleGaugeAdviceConfigurer', + 'io.opentelemetry.extension.incubator.metrics.DoubleHistogramAdviceConfigurer', + 'io.opentelemetry.extension.incubator.metrics.DoubleUpDownCounterAdviceConfigurer', + 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleCounterBuilder', + 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleGaugeBuilder', + 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleUpDownCounterBuilder', + 'io.opentelemetry.extension.incubator.metrics.ExtendedLongCounterBuilder', + 'io.opentelemetry.extension.incubator.metrics.ExtendedLongGaugeBuilder', + 'io.opentelemetry.extension.incubator.metrics.ExtendedLongUpDownCounterBuilder', + 'io.opentelemetry.extension.incubator.metrics.LongCounterAdviceConfigurer', + 'io.opentelemetry.extension.incubator.metrics.LongGauge', + 'io.opentelemetry.extension.incubator.metrics.LongGaugeAdviceConfigurer', + 'io.opentelemetry.extension.incubator.metrics.LongHistogramAdviceConfigurer', + 'io.opentelemetry.extension.incubator.metrics.LongUpDownCounterAdviceConfigurer' ) } diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 deleted file mode 100644 index da3abcc8f70d2..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-api-1.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7ee1ccca95155e4640094ba8dfbd0bb8c1709c83 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..b0ce00e191830 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1 @@ -0,0 +1 @@ +a32dfbd7f01de6711fd0e970f8d4b4c0405056d6 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 deleted file mode 100644 index 2c233d785dcb2..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-1.26.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1b0b6c1a20da0f841634d4f736e331aa4871a4db \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-logs-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-api-logs-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 deleted file mode 100644 index 01d9fd732249b..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-context-1.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42991f523a7a10761213e2f11633c67c8beaed88 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..84cb60a2f7acb --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1 @@ -0,0 +1 @@ +58f665ff01ce6b964cdf0b8cb5cd1c196dfe94ce \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 deleted file mode 100644 index ef07e4cb81e34..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1b932170774da5e766440fa058d879f68fe2c5dd \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..40537a399ab14 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1 @@ -0,0 +1 @@ +58f1a09e89955e6145babf8bcdf80c95174eb817 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 deleted file mode 100644 index dc9946de3b160..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -79a86f258ede8625627e8fbdff07d1149c88a8e6 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..d425ed61cc4cd --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1 @@ -0,0 +1 @@ +4d15a9ea26e8e6ea93287a9f4ee02d91e5a74392 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 deleted file mode 100644 index 2bd3e60a1faf6..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b42359d2232f8d802d55153be5330b1d9e21ee15 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..6b32d98b0f7c7 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1 @@ -0,0 +1 @@ +8e437ba87004bb63069d04fb06beae65b98dd13a \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 deleted file mode 100644 index 90bb8202c4c9d..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.26.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a8abeaee240291cce9067f07569f151d11a6275a \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..13ef6de11e82d --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1 @@ -0,0 +1 @@ +5985d0950746ad12b49cc42c063f26ddfbcaaacb \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 deleted file mode 100644 index 62396a603423f..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8c4af22d7d92a3a79714be3f79724b0ab774ba9e \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..fc5aad9c9011e --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1 @@ -0,0 +1 @@ +b12825541c5dae52a0fb35045c1b36df3ca8f632 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 deleted file mode 100644 index 0fcebee353105..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.26.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fcc5785b2cf2be897f31b927e24b53e46e377388 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..ac522b765da05 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1 @@ -0,0 +1 @@ +4c5531fbc44178a7bcfeb7021ae80e70a7c43458 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 deleted file mode 100644 index 47c7ece8c9f6c..0000000000000 --- a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.26.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1f4f963673f8209208f868666cd43e79b9a2dd15 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1 new file mode 100644 index 0000000000000..089a2484dd1d5 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1 @@ -0,0 +1 @@ +8e8f7a97a4896a81846553275b9d61885be7ef50 \ No newline at end of file From 18ac0606220f8dc5465b1c5d93ae93f3e5b91e63 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Tue, 12 Sep 2023 17:57:58 +0530 Subject: [PATCH 016/388] Block unsafe bootstrap when remote state is enabled (#9965) Signed-off-by: Sooraj Sinha --- .../UnsafeBootstrapAndDetachCommandIT.java | 11 +++++++++++ .../UnsafeBootstrapClusterManagerCommand.java | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java index 4784441058e76..b30eb1f3e3b39 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java @@ -44,6 +44,7 @@ import org.opensearch.env.TestEnvironment; import org.opensearch.gateway.GatewayMetaState; import org.opensearch.gateway.PersistedClusterStateService; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.indices.IndicesService; import org.opensearch.node.Node.DiscoverySettings; import org.opensearch.test.InternalTestCluster; @@ -180,6 +181,16 @@ public void testBootstrapNotClusterManagerEligible() { expectThrows(() -> unsafeBootstrap(environment), UnsafeBootstrapClusterManagerCommand.NOT_CLUSTER_MANAGER_NODE_MSG); } + public void testBootstrapRemoteClusterEnabled() { + final Environment environment = TestEnvironment.newEnvironment( + Settings.builder() + .put(internalCluster().getDefaultSettings()) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build() + ); + expectThrows(() -> unsafeBootstrap(environment), UnsafeBootstrapClusterManagerCommand.REMOTE_CLUSTER_STATE_ENABLED_NODE); + } + public void testBootstrapNoDataFolder() { final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); expectThrows(() -> unsafeBootstrap(environment), OpenSearchNodeCommand.NO_NODE_FOLDER_FOUND_MSG); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/UnsafeBootstrapClusterManagerCommand.java b/server/src/main/java/org/opensearch/cluster/coordination/UnsafeBootstrapClusterManagerCommand.java index 188ea1325e806..168ae5212888f 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/UnsafeBootstrapClusterManagerCommand.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/UnsafeBootstrapClusterManagerCommand.java @@ -53,6 +53,8 @@ import java.util.Locale; import java.util.Objects; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; + /** * Tool to run an unsafe bootstrap * @@ -81,7 +83,11 @@ public class UnsafeBootstrapClusterManagerCommand extends OpenSearchNodeCommand static final Setting UNSAFE_BOOTSTRAP = ClusterService.USER_DEFINED_METADATA.getConcreteSetting( "cluster.metadata.unsafe-bootstrap" ); - + static final String REMOTE_CLUSTER_STATE_ENABLED_NODE = + "Unsafe bootstrap cannot be performed when remote cluster state is enabled. The cluster state in the remote store is considered the source of truth. " + + "In case, you still wish to do best effort recovery with unsafe-bootstrap, then please disable the " + + REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey() + + ". For more details, please check the OpenSearch documentation."; private OptionSpec applyClusterReadOnlyBlockOption; UnsafeBootstrapClusterManagerCommand() { @@ -101,6 +107,13 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) { if (clusterManager == false) { throw new OpenSearchException(NOT_CLUSTER_MANAGER_NODE_MSG); } + // During unsafe bootstrap, node will form a cluster with a new cluster UUID but with the existing metadata. + // This new state will not know about the previous cluster UUIDs and so we will not able to construct + // the cluster UUID chain to get the last known cluster UUID to restore from. + // Blocking unsafe-bootstrap below for this reason. + if (REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) == true) { + throw new OpenSearchException(REMOTE_CLUSTER_STATE_ENABLED_NODE); + } return true; } From 16b54a26010e373bb9c7dce206b98f2d7d073496 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 12 Sep 2023 13:52:28 -0400 Subject: [PATCH 017/388] Don't over-allocate in HeapBufferedAsyncEntityConsumer in order to consume the response (#9993) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../nio/HeapBufferedAsyncEntityConsumer.java | 24 +++++-- .../HeapBufferedAsyncEntityConsumerTests.java | 71 +++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 client/rest/src/test/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d3e6de03709f4..a00bc6447a534 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944)) +- Don't over-allocate in HeapBufferedAsyncEntityConsumer in order to consume the response ([#9993](https://github.com/opensearch-project/OpenSearch/pull/9993)) ### Security diff --git a/client/rest/src/main/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumer.java b/client/rest/src/main/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumer.java index 9bd17d1c24c7e..ae38c1a0308d1 100644 --- a/client/rest/src/main/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumer.java +++ b/client/rest/src/main/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumer.java @@ -86,25 +86,29 @@ protected void data(final ByteBuffer src, final boolean endOfStream) throws IOEx return; } + int len = src.limit(); + if (len < 0) { + len = 4096; + } else if (len > bufferLimitBytes) { + throw new ContentTooLongException( + "entity content is too long [" + len + "] for the configured buffer limit [" + bufferLimitBytes + "]" + ); + } + ByteArrayBuffer buffer = bufferRef.get(); if (buffer == null) { - buffer = new ByteArrayBuffer(bufferLimitBytes); + buffer = new ByteArrayBuffer(len); if (bufferRef.compareAndSet(null, buffer) == false) { buffer = bufferRef.get(); } } - int len = src.limit(); if (buffer.length() + len > bufferLimitBytes) { throw new ContentTooLongException( "entity content is too long [" + len + "] for the configured buffer limit [" + bufferLimitBytes + "]" ); } - if (len < 0) { - len = 4096; - } - if (src.hasArray()) { buffer.append(src.array(), src.arrayOffset() + src.position(), src.remaining()); } else { @@ -136,4 +140,12 @@ public void releaseResources() { buffer = null; } } + + /** + * Gets current byte buffer instance + * @return byte buffer instance + */ + ByteArrayBuffer getBuffer() { + return bufferRef.get(); + } } diff --git a/client/rest/src/test/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumerTests.java b/client/rest/src/test/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumerTests.java new file mode 100644 index 0000000000000..6a4b176edd011 --- /dev/null +++ b/client/rest/src/test/java/org/opensearch/client/nio/HeapBufferedAsyncEntityConsumerTests.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.nio; + +import org.apache.hc.core5.http.ContentTooLongException; +import org.opensearch.client.RestClientTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + +public class HeapBufferedAsyncEntityConsumerTests extends RestClientTestCase { + private static final int BUFFER_LIMIT = 100 * 1024 * 1024 /* 100Mb */; + private HeapBufferedAsyncEntityConsumer consumer; + + @Before + public void setUp() { + consumer = new HeapBufferedAsyncEntityConsumer(BUFFER_LIMIT); + } + + @After + public void tearDown() { + consumer.releaseResources(); + } + + public void testConsumerAllocatesBufferLimit() throws IOException { + consumer.consume(randomByteBufferOfLength(1000).flip()); + assertThat(consumer.getBuffer().capacity(), equalTo(1000)); + } + + public void testConsumerAllocatesEmptyBuffer() throws IOException { + consumer.consume(ByteBuffer.allocate(0).flip()); + assertThat(consumer.getBuffer().capacity(), equalTo(0)); + } + + public void testConsumerExpandsBufferLimits() throws IOException { + consumer.consume(randomByteBufferOfLength(1000).flip()); + consumer.consume(randomByteBufferOfLength(2000).flip()); + consumer.consume(randomByteBufferOfLength(3000).flip()); + assertThat(consumer.getBuffer().capacity(), equalTo(6000)); + } + + public void testConsumerAllocatesLimit() throws IOException { + consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT).flip()); + assertThat(consumer.getBuffer().capacity(), equalTo(BUFFER_LIMIT)); + } + + public void testConsumerFailsToAllocateOverLimit() throws IOException { + assertThrows(ContentTooLongException.class, () -> consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT + 1).flip())); + } + + public void testConsumerFailsToExpandOverLimit() throws IOException { + consumer.consume(randomByteBufferOfLength(BUFFER_LIMIT).flip()); + assertThrows(ContentTooLongException.class, () -> consumer.consume(randomByteBufferOfLength(1).flip())); + } + + private static ByteBuffer randomByteBufferOfLength(int length) { + return ByteBuffer.allocate(length).put(randomBytesOfLength(length)); + } +} From 1aad5a3f1b6a3f4b3521add022b367323a2b327f Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Tue, 12 Sep 2023 10:56:47 -0700 Subject: [PATCH 018/388] Add threadpool wait time metric (#9681) Signed-off-by: Jay Deng --- CHANGELOG.md | 1 + .../test/cat.thread_pool/10_basic.yml | 27 ++++++++++ .../search/stats/ConcurrentSearchStatsIT.java | 53 +++++++++++++++++++ .../OpenSearchThreadPoolExecutor.java | 11 ++++ ...ResizableOpenSearchThreadPoolExecutor.java | 9 ++++ ...eResizingOpenSearchThreadPoolExecutor.java | 8 +++ .../common/util/concurrent/TimedRunnable.java | 8 +++ .../rest/action/cat/RestThreadPoolAction.java | 5 ++ .../org/opensearch/threadpool/ThreadPool.java | 8 +-- .../threadpool/ThreadPoolStats.java | 26 ++++++++- .../cluster/node/stats/NodeStatsTests.java | 3 +- .../threadpool/ThreadPoolStatsTests.java | 24 ++++----- 12 files changed, 166 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a00bc6447a534..9231c67325712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml index 1ce8468cb51f9..9d08f2e4ba53b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml @@ -1,3 +1,30 @@ +"Test cat thread_pool total_wait_time output": + - skip: + version: " - 3.0.0" + reason: thread_pool total_wait_time stats were introduced in V_3.0.0 + + - do: + cat.thread_pool: {} + + - match: + $body: | + / #node_name name active queue rejected + ^ (\S+ \s+ \S+ \s+ \d+ \s+ \d+ \s+ \d+ \n)+ $/ + + - do: + cat.thread_pool: + thread_pool_patterns: search,search_throttled,index_searcher,generic + h: name,total_wait_time,twt + v: true + + - match: + $body: | + /^ id \s+ name \s+ total_wait_time \s+ twt \n + (\S+ \s+ search \s+ \d+s \s+ \d+ \n + \S+ \s+ search_throttled \s+ \d+s \s+ \d+ \n + \S+ \s+ index_searcher \s+ \d+s \s+ \d+ \n + \S+ \s+ generic \s+ -1 \s+ -1 \n)+ $/ + --- "Test cat thread_pool output": - skip: diff --git a/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java index 350f494c35366..f770bd9864850 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java @@ -8,6 +8,8 @@ package org.opensearch.search.stats; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsAction; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.indices.stats.IndexStats; import org.opensearch.action.admin.indices.stats.IndicesStatsRequestBuilder; @@ -26,6 +28,8 @@ import org.opensearch.search.SearchService; import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.threadpool.ThreadPoolStats; import java.util.Arrays; import java.util.Collection; @@ -35,6 +39,7 @@ import java.util.function.Function; import static org.opensearch.index.query.QueryBuilders.scriptQuery; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; @@ -307,6 +312,54 @@ public void testAvgConcurrencyIndexLevel() throws InterruptedException { assertEquals(expectedConcurrency, stats.getTotal().getSearch().getTotal().getConcurrentAvgSliceCount(), 0); } + public void testThreadPoolWaitTime() throws Exception { + int NUM_SHARDS = 1; + String INDEX = "test-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + createIndex( + INDEX, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build() + ); + + ensureGreen(); + + for (int i = 0; i < 10; i++) { + client().prepareIndex(INDEX).setId(Integer.toString(i)).setSource("field", "value" + i).get(); + refresh(); + } + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING.getKey(), 10)) + .execute() + .actionGet(); + + client().prepareSearch(INDEX).execute().actionGet(); + + NodesStatsRequestBuilder nodesStatsRequestBuilder = new NodesStatsRequestBuilder( + client().admin().cluster(), + NodesStatsAction.INSTANCE + ).setNodesIds().all(); + NodesStatsResponse nodesStatsResponse = nodesStatsRequestBuilder.execute().actionGet(); + ThreadPoolStats threadPoolStats = nodesStatsResponse.getNodes().get(0).getThreadPool(); + + for (ThreadPoolStats.Stats stats : threadPoolStats) { + if (stats.getName().equals(ThreadPool.Names.INDEX_SEARCHER)) { + assertThat(stats.getWaitTime().micros(), greaterThan(0L)); + } + } + + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING.getKey(), 2)) + .execute() + .actionGet(); + } + public static class ScriptedDelayedPlugin extends MockScriptPlugin { static final String SCRIPT_NAME = "search_timeout"; diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java index d967b7423ca80..afffec4790873 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchThreadPoolExecutor.java @@ -205,4 +205,15 @@ protected Runnable wrapRunnable(Runnable command) { protected Runnable unwrap(Runnable runnable) { return contextHolder.unwrap(runnable); } + + /** + * Returns the cumulative wait time of the ThreadPool. If the ThreadPool does not support tracking the cumulative pool wait time + * then this should return -1 which will prevent the value from showing up in {@link org.opensearch.threadpool.ThreadPoolStats}. + * ThreadPools that do support this metric should override this method. For example, {@link QueueResizingOpenSearchThreadPoolExecutor} + * does so using the {@link TimedRunnable} to get the difference between Runnable creation and execution. + * + */ + public long getPoolWaitTimeNanos() { + return -1; + } } diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizableOpenSearchThreadPoolExecutor.java b/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizableOpenSearchThreadPoolExecutor.java index 7a0ce8244efe4..c06184f5a5483 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizableOpenSearchThreadPoolExecutor.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizableOpenSearchThreadPoolExecutor.java @@ -9,6 +9,7 @@ package org.opensearch.common.util.concurrent; import org.opensearch.common.ExponentiallyWeightedMovingAverage; +import org.opensearch.common.metrics.CounterMetric; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; @@ -27,6 +28,7 @@ public final class QueueResizableOpenSearchThreadPoolExecutor extends OpenSearch private final ResizableBlockingQueue workQueue; private final Function runnableWrapper; private final ExponentiallyWeightedMovingAverage executionEWMA; + private final CounterMetric poolWaitTime; /** * Create new resizable at runtime thread pool executor @@ -101,6 +103,7 @@ public final class QueueResizableOpenSearchThreadPoolExecutor extends OpenSearch this.workQueue = workQueue; this.runnableWrapper = runnableWrapper; this.executionEWMA = new ExponentiallyWeightedMovingAverage(ewmaAlpha, 0); + this.poolWaitTime = new CounterMetric(); } @Override @@ -156,6 +159,7 @@ protected void afterExecute(Runnable r, Throwable t) { // taskExecutionNanos may be -1 if the task threw an exception executionEWMA.addValue(taskExecutionNanos); } + poolWaitTime.inc(timedRunnable.getWaitTimeNanos()); } /** @@ -173,4 +177,9 @@ public synchronized int resize(int capacity) { capacity ); } + + @Override + public long getPoolWaitTimeNanos() { + return poolWaitTime.count(); + } } diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizingOpenSearchThreadPoolExecutor.java b/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizingOpenSearchThreadPoolExecutor.java index 684dd7c9d8de5..0c0b437e4f390 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizingOpenSearchThreadPoolExecutor.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/QueueResizingOpenSearchThreadPoolExecutor.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.common.ExponentiallyWeightedMovingAverage; +import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.unit.TimeValue; import java.util.Locale; @@ -66,6 +67,7 @@ public final class QueueResizingOpenSearchThreadPoolExecutor extends OpenSearchT private final int maxQueueSize; private final long targetedResponseTimeNanos; private final ExponentiallyWeightedMovingAverage executionEWMA; + private final CounterMetric poolWaitTime; private final AtomicLong totalTaskNanos = new AtomicLong(0); private final AtomicInteger taskCount = new AtomicInteger(0); @@ -97,6 +99,7 @@ public final class QueueResizingOpenSearchThreadPoolExecutor extends OpenSearchT this.maxQueueSize = maxQueueSize; this.targetedResponseTimeNanos = targetedResponseTime.getNanos(); this.executionEWMA = new ExponentiallyWeightedMovingAverage(EWMA_ALPHA, 0); + this.poolWaitTime = new CounterMetric(); logger.debug( "thread pool [{}] will adjust queue by [{}] when determining automatic queue size", getName(), @@ -190,6 +193,7 @@ protected void afterExecute(Runnable r, Throwable t) { // taskExecutionNanos may be -1 if the task threw an exception executionEWMA.addValue(taskExecutionNanos); } + poolWaitTime.inc(timedRunnable.getWaitTimeNanos()); if (taskCount.incrementAndGet() == this.tasksPerFrame) { final long endTimeNs = System.nanoTime(); @@ -290,4 +294,8 @@ protected void appendThreadPoolExecutorDetails(StringBuilder sb) { sb.append("adjustment amount = ").append(QUEUE_ADJUSTMENT_AMOUNT).append(", "); } + @Override + public long getPoolWaitTimeNanos() { + return poolWaitTime.count(); + } } diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/TimedRunnable.java b/server/src/main/java/org/opensearch/common/util/concurrent/TimedRunnable.java index f3bc50a33453b..2eb6657898008 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/TimedRunnable.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/TimedRunnable.java @@ -107,6 +107,14 @@ long getTotalExecutionNanos() { return Math.max(finishTimeNanos - startTimeNanos, 1); } + long getWaitTimeNanos() { + if (startTimeNanos == -1) { + // There must have been an exception thrown, the total time is unknown (-1) + return -1; + } + return Math.max(startTimeNanos - creationTimeNanos, 1); + } + /** * If the task was failed or rejected, return true. * Otherwise, false. diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java index 652bc448144e2..0393dd15c8238 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java @@ -163,6 +163,10 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("rejected", "alias:r;default:true;text-align:right;desc:number of rejected tasks"); table.addCell("largest", "alias:l;default:false;text-align:right;desc:highest number of seen active threads"); table.addCell("completed", "alias:c;default:false;text-align:right;desc:number of completed tasks"); + table.addCell( + "total_wait_time", + "alias:twt;default:false;text-align:right;desc:total time tasks spent waiting in thread_pool queue" + ); table.addCell("core", "alias:cr;default:false;text-align:right;desc:core number of threads in a scaling thread pool"); table.addCell("max", "alias:mx;default:false;text-align:right;desc:maximum number of threads in a scaling thread pool"); table.addCell("size", "alias:sz;default:false;text-align:right;desc:number of threads in a fixed thread pool"); @@ -267,6 +271,7 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR table.addCell(poolStats == null ? null : poolStats.getRejected()); table.addCell(poolStats == null ? null : poolStats.getLargest()); table.addCell(poolStats == null ? null : poolStats.getCompleted()); + table.addCell(poolStats == null ? null : poolStats.getWaitTime()); table.addCell(core); table.addCell(max); table.addCell(size); diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 6ddf3ff6b2f6a..8375ac34972af 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -383,19 +383,21 @@ public ThreadPoolStats stats() { long rejected = -1; int largest = -1; long completed = -1; - if (holder.executor() instanceof ThreadPoolExecutor) { - ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) holder.executor(); + long waitTimeNanos = -1; + if (holder.executor() instanceof OpenSearchThreadPoolExecutor) { + OpenSearchThreadPoolExecutor threadPoolExecutor = (OpenSearchThreadPoolExecutor) holder.executor(); threads = threadPoolExecutor.getPoolSize(); queue = threadPoolExecutor.getQueue().size(); active = threadPoolExecutor.getActiveCount(); largest = threadPoolExecutor.getLargestPoolSize(); completed = threadPoolExecutor.getCompletedTaskCount(); + waitTimeNanos = threadPoolExecutor.getPoolWaitTimeNanos(); RejectedExecutionHandler rejectedExecutionHandler = threadPoolExecutor.getRejectedExecutionHandler(); if (rejectedExecutionHandler instanceof XRejectedExecutionHandler) { rejected = ((XRejectedExecutionHandler) rejectedExecutionHandler).rejected(); } } - stats.add(new ThreadPoolStats.Stats(name, threads, queue, active, rejected, largest, completed)); + stats.add(new ThreadPoolStats.Stats(name, threads, queue, active, rejected, largest, completed, waitTimeNanos)); } return new ThreadPoolStats(stats); } diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java index b4d7e4a3fbf7a..8b1e1461d8b02 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java @@ -32,6 +32,8 @@ package org.opensearch.threadpool; +import org.opensearch.Version; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -65,8 +67,9 @@ public static class Stats implements Writeable, ToXContentFragment, Comparable stats = new ArrayList<>(); - stats.add(new ThreadPoolStats.Stats("z", -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats("m", 3, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats("m", 1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats("d", -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats("m", 2, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats("t", -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats("a", -1, 0, 0, 0, 0, 0L)); + stats.add(new ThreadPoolStats.Stats("z", -1, 0, 0, 0, 0, 0L, 0L)); + stats.add(new ThreadPoolStats.Stats("m", 3, 0, 0, 0, 0, 0L, 0L)); + stats.add(new ThreadPoolStats.Stats("m", 1, 0, 0, 0, 0, 0L, 0L)); + stats.add(new ThreadPoolStats.Stats("d", -1, 0, 0, 0, 0, 0L, 0L)); + stats.add(new ThreadPoolStats.Stats("m", 2, 0, 0, 0, 0, 0L, 0L)); + stats.add(new ThreadPoolStats.Stats("t", -1, 0, 0, 0, 0, 0L, 0L)); + stats.add(new ThreadPoolStats.Stats("a", -1, 0, 0, 0, 0, 0L, 0L)); List copy = new ArrayList<>(stats); Collections.sort(copy); @@ -79,11 +79,11 @@ public void testThreadPoolStatsToXContent() throws IOException { try (BytesStreamOutput os = new BytesStreamOutput()) { List stats = new ArrayList<>(); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.WARMER, -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.FORCE_MERGE, -1, 0, 0, 0, 0, 0L)); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SAME, -1, 0, 0, 0, 0, 0L)); + stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, -1, 0, 0, 0, 0, 0L, 0L)); + stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.WARMER, -1, 0, 0, 0, 0, 0L, -1L)); + stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, -1, 0, 0, 0, 0, 0L, -1L)); + stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.FORCE_MERGE, -1, 0, 0, 0, 0, 0L, -1L)); + stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SAME, -1, 0, 0, 0, 0, 0L, -1L)); ThreadPoolStats threadPoolStats = new ThreadPoolStats(stats); try (XContentBuilder builder = new XContentBuilder(MediaTypeRegistry.JSON.xContent(), os)) { From c261cff2303087a48ed9482f1063c6f04d48181c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 11:35:08 -0700 Subject: [PATCH 019/388] Bump com.google.cloud:google-cloud-core-http from 2.21.1 to 2.23.0 in /plugins/repository-gcs (#9971) * Bump com.google.cloud:google-cloud-core-http in /plugins/repository-gcs Bumps com.google.cloud:google-cloud-core-http from 2.21.1 to 2.23.0. --- updated-dependencies: - dependency-name: com.google.cloud:google-cloud-core-http dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- .../licenses/google-cloud-core-http-2.21.1.jar.sha1 | 1 - .../licenses/google-cloud-core-http-2.23.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9231c67325712..cdf488edb41f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968)) - Bump OpenTelemetry from 1.26.0 to 1.30.1 ([#9950](https://github.com/opensearch-project/OpenSearch/pull/9950)) - Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973](https://github.com/opensearch-project/OpenSearch/pull/9973)) +- Bump `com.google.cloud:google-cloud-core-http` from 2.21.1 to 2.23.0 ([#9971](https://github.com/opensearch-project/OpenSearch/pull/9971)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index e7ee980114e5e..1f73d6a4d7520 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -67,7 +67,7 @@ dependencies { api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" api 'com.google.cloud:google-cloud-core:2.5.10' - api 'com.google.cloud:google-cloud-core-http:2.21.1' + api 'com.google.cloud:google-cloud-core-http:2.23.0' api 'com.google.cloud:google-cloud-storage:1.113.1' api 'com.google.code.gson:gson:2.10.1' diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 deleted file mode 100644 index cc5e7a53098ac..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -88dd2b413dd06826c611e39e6e3259e069f02f66 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 new file mode 100644 index 0000000000000..9db3cbcbec35b --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 @@ -0,0 +1 @@ +9913d0806fcfbfbc4a775f29865126ed8465464b \ No newline at end of file From 0566926c5e8cae5ea251ddfd4c5648cd3f3c8994 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 12 Sep 2023 15:15:22 -0400 Subject: [PATCH 020/388] Remove tf() painless method (#9995) Signed-off-by: Louis Chu --- .../painless/spi/org.opensearch.score.txt | 1 - .../TermFrequencyFunctionFactory.java | 11 ---------- .../opensearch/script/ScoreScriptUtils.java | 22 ------------------- 3 files changed, 34 deletions(-) diff --git a/modules/lang-painless/src/main/resources/org/opensearch/painless/spi/org.opensearch.score.txt b/modules/lang-painless/src/main/resources/org/opensearch/painless/spi/org.opensearch.score.txt index 5533f0bc55522..9bce617099c6f 100644 --- a/modules/lang-painless/src/main/resources/org/opensearch/painless/spi/org.opensearch.score.txt +++ b/modules/lang-painless/src/main/resources/org/opensearch/painless/spi/org.opensearch.score.txt @@ -24,7 +24,6 @@ class org.opensearch.script.ScoreScript @no_import { static_import { int termFreq(org.opensearch.script.ScoreScript, String, String) bound_to org.opensearch.script.ScoreScriptUtils$TermFreq - float tf(org.opensearch.script.ScoreScript, String, String) bound_to org.opensearch.script.ScoreScriptUtils$TF long totalTermFreq(org.opensearch.script.ScoreScript, String, String) bound_to org.opensearch.script.ScoreScriptUtils$TotalTermFreq long sumTotalTermFreq(org.opensearch.script.ScoreScript, String) bound_to org.opensearch.script.ScoreScriptUtils$SumTotalTermFreq double saturation(double, double) from_class org.opensearch.script.ScoreScriptUtils diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/TermFrequencyFunctionFactory.java b/server/src/main/java/org/opensearch/index/query/functionscore/TermFrequencyFunctionFactory.java index 4edcd34889abd..9db58f0f78a30 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/TermFrequencyFunctionFactory.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/TermFrequencyFunctionFactory.java @@ -11,7 +11,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.valuesource.SumTotalTermFreqValueSource; -import org.apache.lucene.queries.function.valuesource.TFValueSource; import org.apache.lucene.queries.function.valuesource.TermFreqValueSource; import org.apache.lucene.queries.function.valuesource.TotalTermFreqValueSource; import org.apache.lucene.search.IndexSearcher; @@ -42,15 +41,6 @@ public static TermFrequencyFunction createFunction( TermFreqValueSource termFreqValueSource = new TermFreqValueSource(field, term, field, BytesRefs.toBytesRef(term)); FunctionValues functionValues = termFreqValueSource.getValues(null, readerContext); return docId -> functionValues.intVal(docId); - case TF: - TFValueSource tfValueSource = new TFValueSource(field, term, field, BytesRefs.toBytesRef(term)); - Map tfContext = new HashMap<>() { - { - put("searcher", indexSearcher); - } - }; - functionValues = tfValueSource.getValues(tfContext, readerContext); - return docId -> functionValues.floatVal(docId); case TOTAL_TERM_FREQ: TotalTermFreqValueSource totalTermFreqValueSource = new TotalTermFreqValueSource( field, @@ -78,7 +68,6 @@ public static TermFrequencyFunction createFunction( */ public enum TermFrequencyFunctionName { TERM_FREQ("termFreq"), - TF("tf"), TOTAL_TERM_FREQ("totalTermFreq"), SUM_TOTAL_TERM_FREQ("sumTotalTermFreq"); diff --git a/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java b/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java index 0767c29fa1b31..378ff1ccfe4a8 100644 --- a/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java +++ b/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java @@ -50,7 +50,6 @@ import static org.opensearch.common.util.BitMixer.mix32; import static org.opensearch.index.query.functionscore.TermFrequencyFunctionFactory.TermFrequencyFunctionName.SUM_TOTAL_TERM_FREQ; import static org.opensearch.index.query.functionscore.TermFrequencyFunctionFactory.TermFrequencyFunctionName.TERM_FREQ; -import static org.opensearch.index.query.functionscore.TermFrequencyFunctionFactory.TermFrequencyFunctionName.TF; import static org.opensearch.index.query.functionscore.TermFrequencyFunctionFactory.TermFrequencyFunctionName.TOTAL_TERM_FREQ; /** @@ -95,27 +94,6 @@ public int termFreq(String field, String term) { } } - /** - * Calculates the term frequency-inverse document frequency (tf-idf) for a specific term within a field. - * - * @opensearch.internal - */ - public static final class TF { - private final ScoreScript scoreScript; - - public TF(ScoreScript scoreScript) { - this.scoreScript = scoreScript; - } - - public float tf(String field, String term) { - try { - return (float) scoreScript.getTermFrequency(TF, field, term); - } catch (Exception e) { - throw ExceptionsHelper.convertToOpenSearchException(e); - } - } - } - /** * Retrieves the total term frequency within a field for a specific term. * From 7e070230436654c730bc7c51bed1603ffc9b99fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:24:24 -0400 Subject: [PATCH 021/388] Bump org.apache.commons:commons-compress in /test/fixtures/hdfs-fixture (#9972) Bumps org.apache.commons:commons-compress from 1.23.0 to 1.24.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-compress dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdf488edb41f1..425177bfd2cf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,7 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) - Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968)) - Bump OpenTelemetry from 1.26.0 to 1.30.1 ([#9950](https://github.com/opensearch-project/OpenSearch/pull/9950)) -- Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973](https://github.com/opensearch-project/OpenSearch/pull/9973)) +- Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973, #9972](https://github.com/opensearch-project/OpenSearch/pull/9973, https://github.com/opensearch-project/OpenSearch/pull/9972)) - Bump `com.google.cloud:google-cloud-core-http` from 2.21.1 to 2.23.0 ([#9971](https://github.com/opensearch-project/OpenSearch/pull/9971)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 79841ca70b53a..42de21875d291 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -50,7 +50,7 @@ dependencies { exclude module: "json-io" } api "org.codehaus.jettison:jettison:${versions.jettison}" - api "org.apache.commons:commons-compress:1.23.0" + api "org.apache.commons:commons-compress:${versions.commonscompress}" api "commons-codec:commons-codec:${versions.commonscodec}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api "io.netty:netty-all:${versions.netty}" From 443cfcaeaa009f82aa544ee7e4b131ffa147a2df Mon Sep 17 00:00:00 2001 From: Neetika Singhal Date: Tue, 12 Sep 2023 13:36:20 -0700 Subject: [PATCH 022/388] Parameterize the tests for concurrent search (#9845) Signed-off-by: Neetika Singhal --- .../search/SearchWithRejectionsIT.java | 27 ++++++++++++++++- .../CardinalityWithRequestBreakerIT.java | 27 +++++++++++++++-- .../aggregations/metrics/GeoCentroidIT.java | 5 ++++ .../backpressure/SearchBackpressureIT.java | 25 +++++++++++++++- .../search/basic/SearchRedStateIndexIT.java | 26 +++++++++++++++- .../basic/SearchWhileCreatingIndexIT.java | 30 +++++++++++++++++-- .../search/basic/SearchWhileRelocatingIT.java | 26 +++++++++++++++- .../basic/SearchWithRandomExceptionsIT.java | 24 ++++++++++++++- .../basic/SearchWithRandomIOExceptionsIT.java | 24 ++++++++++++++- .../basic/TransportSearchFailuresIT.java | 29 ++++++++++++++++-- .../basic/TransportTwoNodesSearchIT.java | 27 +++++++++++++++-- .../search/fetch/FetchSubPhasePluginIT.java | 28 +++++++++++++++-- .../search/fetch/subphase/InnerHitsIT.java | 25 ++++++++++++++-- .../fetch/subphase/MatchedQueriesIT.java | 30 +++++++++++++++++-- .../highlight/CustomHighlighterSearchIT.java | 26 ++++++++++++++-- .../highlight/HighlighterSearchIT.java | 25 ++++++++++++++-- .../search/fieldcaps/FieldCapabilitiesIT.java | 26 ++++++++++++++-- .../search/fields/SearchFieldsIT.java | 25 ++++++++++++++-- .../functionscore/DecayFunctionScoreIT.java | 27 +++++++++++++++-- .../functionscore/ExplainableScriptIT.java | 25 ++++++++++++++-- .../FunctionScoreFieldValueIT.java | 29 ++++++++++++++++-- .../search/functionscore/FunctionScoreIT.java | 27 +++++++++++++++-- .../functionscore/FunctionScorePluginIT.java | 27 +++++++++++++++-- .../search/functionscore/QueryRescorerIT.java | 27 +++++++++++++++-- .../functionscore/RandomScoreFunctionIT.java | 26 ++++++++++++++-- .../opensearch/search/geo/GeoFilterIT.java | 27 +++++++++++++++-- .../opensearch/search/geo/GeoPolygonIT.java | 26 +++++++++++++++- .../search/geo/GeoShapeIntegrationIT.java | 28 +++++++++++++++-- .../geo/LegacyGeoShapeIntegrationIT.java | 27 +++++++++++++++-- .../search/morelikethis/MoreLikeThisIT.java | 26 ++++++++++++++-- .../search/msearch/MultiSearchIT.java | 29 ++++++++++++++++-- .../search/nested/SimpleNestedIT.java | 29 ++++++++++++++++-- .../search/preference/SearchPreferenceIT.java | 26 +++++++++++++++- .../aggregation/AggregationProfilerIT.java | 28 ++++++++++++++++- .../org/opensearch/search/query/ExistsIT.java | 28 +++++++++++++++-- .../search/query/MultiMatchQueryIT.java | 25 ++++++++++++++-- .../search/query/QueryStringIT.java | 27 +++++++++++++++-- .../search/query/ScriptScoreQueryIT.java | 26 ++++++++++++++-- .../search/query/SearchQueryIT.java | 25 ++++++++++++++-- .../search/query/SimpleQueryStringIT.java | 26 ++++++++++++++-- .../search/slice/SearchSliceIT.java | 27 +++++++++++++++-- .../search/source/MetadataFetchingIT.java | 29 ++++++++++++++++-- .../search/source/SourceFetchingIT.java | 30 +++++++++++++++++-- .../metrics/AbstractGeoTestCase.java | 26 +++++++++++++++- 44 files changed, 1082 insertions(+), 76 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchWithRejectionsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchWithRejectionsIT.java index 87c6aa2202ff5..24c72a66da6d0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchWithRejectionsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchWithRejectionsIT.java @@ -32,20 +32,45 @@ package org.opensearch.search; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class SearchWithRejectionsIT extends OpenSearchIntegTestCase { +public class SearchWithRejectionsIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchWithRejectionsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public Settings nodeSettings(int nodeOrdinal) { return Settings.builder() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java index a3221e128a5dc..2bf5230c67b43 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java @@ -32,23 +32,46 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.BucketOrder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; import java.util.Map; import java.util.stream.IntStream; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.cardinality; import static org.opensearch.search.aggregations.AggregationBuilders.terms; -public class CardinalityWithRequestBreakerIT extends OpenSearchIntegTestCase { +public class CardinalityWithRequestBreakerIT extends ParameterizedOpenSearchIntegTestCase { + + public CardinalityWithRequestBreakerIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } /** * Test that searches using cardinality aggregations returns all request breaker memory. diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java index ffc31b7cdb7c4..78100d1778ecf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java @@ -34,6 +34,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.settings.Settings; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.test.OpenSearchIntegTestCase; @@ -54,6 +55,10 @@ public class GeoCentroidIT extends AbstractGeoTestCase { private static final String aggName = "geoCentroid"; + public GeoCentroidIT(Settings dynamicSettings) { + super(dynamicSettings); + } + public void testEmptyAggregation() throws Exception { SearchResponse response = client().prepareSearch(EMPTY_IDX_NAME) .setQuery(matchAllQuery()) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java index 8eb5167ca9e51..28ada82a1c56b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java @@ -8,6 +8,8 @@ package org.opensearch.search.backpressure; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; @@ -19,6 +21,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; @@ -34,6 +37,7 @@ import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import org.hamcrest.MatcherAssert; @@ -42,6 +46,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -50,16 +55,34 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class SearchBackpressureIT extends OpenSearchIntegTestCase { +public class SearchBackpressureIT extends ParameterizedOpenSearchIntegTestCase { private static final TimeValue TIMEOUT = new TimeValue(10, TimeUnit.SECONDS); private static final int MOVING_AVERAGE_WINDOW_SIZE = 10; + public SearchBackpressureIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { final List> plugins = new ArrayList<>(super.nodePlugins()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java index d88893d1bcd71..44c4981dfdb36 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.basic; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; @@ -40,13 +42,18 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchService; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.After; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -54,7 +61,24 @@ import static org.hamcrest.Matchers.lessThan; @OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) -public class SearchRedStateIndexIT extends OpenSearchIntegTestCase { +public class SearchRedStateIndexIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchRedStateIndexIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public void testAllowPartialsWithRedState() throws Exception { final int numShards = cluster().numDataNodes() + 2; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java index 6099c5342a9d3..71af7215c4eb7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java @@ -32,13 +32,21 @@ package org.opensearch.search.basic; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.refresh.RefreshResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -46,7 +54,25 @@ * This test basically verifies that search with a single shard active (cause we indexed to it) and other * shards possibly not active at all (cause they haven't allocated) will still work. */ -public class SearchWhileCreatingIndexIT extends OpenSearchIntegTestCase { +public class SearchWhileCreatingIndexIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchWhileCreatingIndexIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testIndexCausesIndexCreation() throws Exception { searchWhileCreatingIndex(false, 1); // 1 replica in our default... } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileRelocatingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileRelocatingIT.java index 1f1384cc5f72d..6d2ec845afa98 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileRelocatingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileRelocatingIT.java @@ -32,28 +32,52 @@ package org.opensearch.search.basic; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.SearchHits; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoTimeout; import static org.opensearch.test.hamcrest.OpenSearchAssertions.formatShardStatus; import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) -public class SearchWhileRelocatingIT extends OpenSearchIntegTestCase { +public class SearchWhileRelocatingIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchWhileRelocatingIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public void testSearchAndRelocateConcurrentlyRandomReplicas() throws Exception { testSearchAndRelocateConcurrently(randomIntBetween(0, 1)); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java index b6da477d1b23e..aa82b9d21c7fb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.basic; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.LeafReader; @@ -47,12 +49,14 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.Settings.Builder; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.MockEngineFactoryPlugin; import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.Plugin; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.engine.MockEngineSupport; import org.opensearch.test.engine.ThrowingLeafReaderWrapper; @@ -64,9 +68,27 @@ import java.util.Random; import java.util.concurrent.ExecutionException; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; -public class SearchWithRandomExceptionsIT extends OpenSearchIntegTestCase { +public class SearchWithRandomExceptionsIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchWithRandomExceptionsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomIOExceptionsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomIOExceptionsIT.java index 0bcd945ba47b3..446a0bce58d66 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomIOExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomIOExceptionsIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.basic; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.tests.util.English; import org.opensearch.OpenSearchException; import org.opensearch.action.DocWriteResponse; @@ -43,11 +45,13 @@ import org.opensearch.client.Requests; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.Plugin; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.store.MockFSDirectoryFactory; import org.opensearch.test.store.MockFSIndexStore; @@ -56,10 +60,28 @@ import java.util.Collection; import java.util.concurrent.ExecutionException; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; -public class SearchWithRandomIOExceptionsIT extends OpenSearchIntegTestCase { +public class SearchWithRandomIOExceptionsIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchWithRandomIOExceptionsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportSearchFailuresIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportSearchFailuresIT.java index 841821b5bbad6..cbe52abf5279b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportSearchFailuresIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportSearchFailuresIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.basic; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.refresh.RefreshResponse; @@ -41,23 +43,46 @@ import org.opensearch.client.Requests; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.Priority; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.client.Requests.clusterHealthRequest; import static org.opensearch.client.Requests.refreshRequest; import static org.opensearch.client.Requests.searchRequest; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class TransportSearchFailuresIT extends OpenSearchIntegTestCase { +public class TransportSearchFailuresIT extends ParameterizedOpenSearchIntegTestCase { + + public TransportSearchFailuresIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected int maximumNumberOfReplicas() { return 1; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java index 069559d6d11b1..ce5f3f63faa66 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.basic; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.action.search.SearchPhaseExecutionException; @@ -39,6 +41,7 @@ import org.opensearch.client.Requests; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -51,9 +54,11 @@ import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Set; import java.util.TreeSet; @@ -66,6 +71,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @@ -74,7 +80,24 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; -public class TransportTwoNodesSearchIT extends OpenSearchIntegTestCase { +public class TransportTwoNodesSearchIT extends ParameterizedOpenSearchIntegTestCase { + + public TransportTwoNodesSearchIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected int numberOfReplicas() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java index 127bd3176453b..86df25c4dad65 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.fetch; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; @@ -39,6 +41,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.document.DocumentField; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -48,12 +52,13 @@ import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.SearchExtBuilder; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -64,11 +69,30 @@ import static java.util.Collections.singletonList; import static org.opensearch.client.Requests.indexRequest; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.CoreMatchers.equalTo; @ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2) -public class FetchSubPhasePluginIT extends OpenSearchIntegTestCase { +public class FetchSubPhasePluginIT extends ParameterizedOpenSearchIntegTestCase { + + public FetchSubPhasePluginIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singletonList(FetchTermVectorsPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java index f43918e28b9b5..9b3e1337418cc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.fetch.subphase; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.util.ArrayUtil; import org.opensearch.action.index.IndexRequestBuilder; @@ -39,6 +41,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.query.BoolQueryBuilder; @@ -55,7 +58,7 @@ import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.InternalSettingsPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; import java.util.Arrays; @@ -72,6 +75,7 @@ import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.index.query.QueryBuilders.nestedQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -84,7 +88,24 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class InnerHitsIT extends OpenSearchIntegTestCase { +public class InnerHitsIT extends ParameterizedOpenSearchIntegTestCase { + + public InnerHitsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java index e9cad63cbac94..23b5d0cab0697 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java @@ -32,7 +32,11 @@ package org.opensearch.search.fetch.subphase; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentHelper; @@ -41,7 +45,10 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.search.SearchHit; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.constantScoreQuery; @@ -52,11 +59,30 @@ import static org.opensearch.index.query.QueryBuilders.termQuery; import static org.opensearch.index.query.QueryBuilders.termsQuery; import static org.opensearch.index.query.QueryBuilders.wrapperQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItemInArray; -public class MatchedQueriesIT extends OpenSearchIntegTestCase { +public class MatchedQueriesIT extends ParameterizedOpenSearchIntegTestCase { + + public MatchedQueriesIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testSimpleMatchedQueryFromFilteredQuery() throws Exception { createIndex("test"); ensureGreen(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java index 7df5b9b88a69c..fe17c3e22d43c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java @@ -31,12 +31,16 @@ package org.opensearch.search.fetch.subphase.highlight; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.Before; import java.io.IOException; @@ -45,6 +49,7 @@ import java.util.HashMap; import java.util.Map; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHighlight; import static org.hamcrest.Matchers.equalTo; @@ -52,7 +57,24 @@ * Integration test for highlighters registered by a plugin. */ @ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) -public class CustomHighlighterSearchIT extends OpenSearchIntegTestCase { +public class CustomHighlighterSearchIT extends ParameterizedOpenSearchIntegTestCase { + + public CustomHighlighterSearchIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 2e70029cfb9f6..4cdf5ae8e674f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -31,6 +31,7 @@ package org.opensearch.search.fetch.subphase.highlight; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.analysis.Analyzer; @@ -48,6 +49,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.Settings.Builder; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; @@ -73,7 +75,7 @@ import org.opensearch.search.sort.SortOrder; import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.MockKeywordPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; @@ -107,6 +109,7 @@ import static org.opensearch.index.query.QueryBuilders.regexpQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; import static org.opensearch.index.query.QueryBuilders.wildcardQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.builder.SearchSourceBuilder.highlight; import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -125,10 +128,28 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; -public class HighlighterSearchIT extends OpenSearchIntegTestCase { +public class HighlighterSearchIT extends ParameterizedOpenSearchIntegTestCase { + // TODO as we move analyzers out of the core we need to move some of these into HighlighterWithAnalyzersTests private static final String[] ALL_TYPES = new String[] { "plain", "fvh", "unified" }; + public HighlighterSearchIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Arrays.asList(InternalSettingsPlugin.class, MockKeywordPlugin.class, MockAnalysisPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java index 21efcd738ae9f..f5d1b8234558e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -32,15 +32,19 @@ package org.opensearch.search.fieldcaps; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.fieldcaps.FieldCapabilities; import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse; import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.MapperPlugin; import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.Before; import java.util.ArrayList; @@ -52,9 +56,27 @@ import java.util.function.Function; import java.util.function.Predicate; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; -public class FieldCapabilitiesIT extends OpenSearchIntegTestCase { +public class FieldCapabilitiesIT extends ParameterizedOpenSearchIntegTestCase { + + public FieldCapabilitiesIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Before public void setUp() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java index 65798713bb577..799bbf91a567d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.fields; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; @@ -41,6 +43,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateUtils; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.common.bytes.BytesArray; @@ -60,7 +63,7 @@ import org.opensearch.search.lookup.FieldLookup; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.InternalSettingsPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -88,6 +91,7 @@ import static org.opensearch.common.util.set.Sets.newHashSet; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -100,7 +104,24 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class SearchFieldsIT extends OpenSearchIntegTestCase { +public class SearchFieldsIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchFieldsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java index c888dcddb5611..6eb528e0bb7d3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.functionscore; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.Version; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; @@ -44,6 +46,7 @@ import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.common.lucene.search.function.FunctionScoreQuery.ScoreMode; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -53,12 +56,14 @@ import org.opensearch.search.MultiValueMode; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Locale; @@ -72,6 +77,7 @@ import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.exponentialDecayFunction; import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.gaussDecayFunction; import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.linearDecayFunction; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @@ -85,7 +91,24 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; -public class DecayFunctionScoreIT extends OpenSearchIntegTestCase { +public class DecayFunctionScoreIT extends ParameterizedOpenSearchIntegTestCase { + + public DecayFunctionScoreIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected boolean forbidPrivateIndexSettings() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java index f329677a94340..62d0d89c644a5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.functionscore; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -41,6 +43,7 @@ import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.lucene.search.function.Functions; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.ScriptPlugin; @@ -53,9 +56,9 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.lookup.SearchLookup; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.io.IOException; @@ -73,13 +76,31 @@ import static org.opensearch.index.query.QueryBuilders.functionScoreQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) -public class ExplainableScriptIT extends OpenSearchIntegTestCase { +public class ExplainableScriptIT extends ParameterizedOpenSearchIntegTestCase { + + public ExplainableScriptIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public static class ExplainableScriptPlugin extends Plugin implements ScriptPlugin { @Override diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java index cf133396e6fcb..b09914c4aa764 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java @@ -32,19 +32,26 @@ package org.opensearch.search.functionscore; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.SearchHit; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.functionScoreQuery; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertOrderedSearchHits; @@ -54,7 +61,25 @@ /** * Tests for the {@code field_value_factor} function in a function_score query. */ -public class FunctionScoreFieldValueIT extends OpenSearchIntegTestCase { +public class FunctionScoreFieldValueIT extends ParameterizedOpenSearchIntegTestCase { + + public FunctionScoreFieldValueIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testFieldValueFactor() throws IOException { assertAcked( prepareCreate("test").setMapping( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java index 3d24933f66d17..88395f25700d2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java @@ -32,10 +32,14 @@ package org.opensearch.search.functionscore; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder; @@ -45,11 +49,12 @@ import org.opensearch.script.ScriptType; import org.opensearch.search.SearchHit; import org.opensearch.search.aggregations.bucket.terms.Terms; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -63,6 +68,7 @@ import static org.opensearch.index.query.QueryBuilders.functionScoreQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -72,11 +78,28 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -public class FunctionScoreIT extends OpenSearchIntegTestCase { +public class FunctionScoreIT extends ParameterizedOpenSearchIntegTestCase { static final String TYPE = "type"; static final String INDEX = "index"; + public FunctionScoreIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java index 384d9f2c61042..1df4acac0dcf0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java @@ -32,11 +32,15 @@ package org.opensearch.search.functionscore; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.Explanation; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.common.Priority; import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.query.functionscore.DecayFunction; @@ -46,9 +50,9 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.SearchHits; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.io.IOException; @@ -62,11 +66,30 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.functionScoreQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; import static org.hamcrest.Matchers.equalTo; @ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) -public class FunctionScorePluginIT extends OpenSearchIntegTestCase { +public class FunctionScorePluginIT extends ParameterizedOpenSearchIntegTestCase { + + public FunctionScorePluginIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Arrays.asList(CustomDistanceScorePlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java index 34a304615b075..de4c85301547c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.functionscore; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.Explanation; import org.apache.lucene.tests.util.English; import org.opensearch.action.index.IndexRequestBuilder; @@ -41,6 +43,7 @@ import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.Settings.Builder; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.Operator; @@ -52,9 +55,10 @@ import org.opensearch.search.rescore.QueryRescoreMode; import org.opensearch.search.rescore.QueryRescorerBuilder; import org.opensearch.search.sort.SortBuilders; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; @@ -70,6 +74,7 @@ import static org.opensearch.index.query.QueryBuilders.queryStringQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.weightFactorFunction; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFirstHit; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFourthHit; @@ -86,7 +91,25 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; -public class QueryRescorerIT extends OpenSearchIntegTestCase { +public class QueryRescorerIT extends ParameterizedOpenSearchIntegTestCase { + + public QueryRescorerIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testEnforceWindowSize() { createIndex("test"); // this diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java index 2176b93079d02..8f43cefd2d53b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java @@ -31,8 +31,12 @@ package org.opensearch.search.functionscore; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.util.ArrayUtil; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; @@ -43,7 +47,7 @@ import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.search.SearchHit; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.CoreMatchers; import java.util.Arrays; @@ -60,6 +64,7 @@ import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.randomFunction; import static org.opensearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; import static org.opensearch.script.MockScriptPlugin.NAME; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.allOf; @@ -71,7 +76,24 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; -public class RandomScoreFunctionIT extends OpenSearchIntegTestCase { +public class RandomScoreFunctionIT extends ParameterizedOpenSearchIntegTestCase { + + public RandomScoreFunctionIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java index 9c06082db31d4..00524c6e04707 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.geo; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; @@ -54,6 +56,7 @@ import org.opensearch.common.geo.builders.PointBuilder; import org.opensearch.common.geo.builders.PolygonBuilder; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; @@ -61,7 +64,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; import org.junit.BeforeClass; @@ -70,6 +73,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Random; import java.util.zip.GZIPInputStream; @@ -84,6 +89,7 @@ import static org.opensearch.index.query.QueryBuilders.geoDistanceQuery; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.matchQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFirstHit; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.hasId; @@ -93,7 +99,24 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class GeoFilterIT extends OpenSearchIntegTestCase { +public class GeoFilterIT extends ParameterizedOpenSearchIntegTestCase { + + public GeoFilterIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected boolean forbidPrivateIndexSettings() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoPolygonIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoPolygonIT.java index d0b017732b270..85cb087585d31 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoPolygonIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoPolygonIT.java @@ -32,28 +32,52 @@ package org.opensearch.search.geo; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.Version; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.SearchHit; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.geoPolygonQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoPolygonIT extends OpenSearchIntegTestCase { +public class GeoPolygonIT extends ParameterizedOpenSearchIntegTestCase { + + public GeoPolygonIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected boolean forbidPrivateIndexSettings() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoShapeIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoShapeIntegrationIT.java index 98e9b35208f3e..1f9b6ae434f75 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoShapeIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoShapeIntegrationIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.geo; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.ClusterState; @@ -39,22 +41,44 @@ import org.opensearch.common.geo.builders.PointBuilder; import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.GeoShapeFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.indices.IndicesService; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.index.query.QueryBuilders.geoShapeQuery; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class GeoShapeIntegrationIT extends OpenSearchIntegTestCase { +public class GeoShapeIntegrationIT extends ParameterizedOpenSearchIntegTestCase { + + public GeoShapeIntegrationIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/LegacyGeoShapeIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/LegacyGeoShapeIntegrationIT.java index 4ff7f49082901..d21d6036c9673 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/LegacyGeoShapeIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/LegacyGeoShapeIntegrationIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.geo; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.search.SearchResponse; @@ -39,6 +41,7 @@ import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.geo.builders.ShapeBuilder; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -47,17 +50,37 @@ import org.opensearch.index.mapper.LegacyGeoShapeFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.indices.IndicesService; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.index.query.QueryBuilders.geoShapeQuery; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class LegacyGeoShapeIntegrationIT extends OpenSearchIntegTestCase { +public class LegacyGeoShapeIntegrationIT extends ParameterizedOpenSearchIntegTestCase { + + public LegacyGeoShapeIntegrationIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } /** * Test that orientation parameter correctly persists across cluster restart diff --git a/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java b/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java index 4ac8eaa8a1121..87435bb0bd09d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.morelikethis; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.RoutingMissingException; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.action.index.IndexRequestBuilder; @@ -39,6 +41,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.MoreLikeThisQueryBuilder; @@ -47,10 +50,11 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.Plugin; import org.opensearch.test.InternalSettingsPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -63,6 +67,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.moreLikeThisQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @@ -74,7 +79,24 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; -public class MoreLikeThisIT extends OpenSearchIntegTestCase { +public class MoreLikeThisIT extends ParameterizedOpenSearchIntegTestCase { + + public MoreLikeThisIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java index 9c2ddbba89903..bc1d2833ecbbf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java @@ -32,19 +32,44 @@ package org.opensearch.search.msearch; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.MultiSearchRequest; import org.opensearch.action.search.MultiSearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFirstHit; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; import static org.opensearch.test.hamcrest.OpenSearchAssertions.hasId; import static org.hamcrest.Matchers.equalTo; -public class MultiSearchIT extends OpenSearchIntegTestCase { +public class MultiSearchIT extends ParameterizedOpenSearchIntegTestCase { + + public MultiSearchIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public void testSimpleMultiSearch() { createIndex("test"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java index 2db4121144bca..83dec7b27a897 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.nested; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.Explanation; import org.apache.lucene.search.join.ScoreMode; import org.opensearch.action.DocWriteResponse; @@ -45,6 +47,7 @@ import org.opensearch.action.search.SearchType; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -53,7 +56,10 @@ import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortMode; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; @@ -61,6 +67,7 @@ import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.nestedQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @@ -70,7 +77,25 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; -public class SimpleNestedIT extends OpenSearchIntegTestCase { +public class SimpleNestedIT extends ParameterizedOpenSearchIntegTestCase { + + public SimpleNestedIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testSimpleNested() throws Exception { assertAcked(prepareCreate("test").setMapping("nested1", "type=nested")); ensureGreen(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java index 911d2fcae01fe..425764b1c88d2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.preference; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.search.SearchRequestBuilder; @@ -42,19 +44,24 @@ import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.node.Node; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.Set; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -64,7 +71,24 @@ import static org.hamcrest.Matchers.not; @OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) -public class SearchPreferenceIT extends OpenSearchIntegTestCase { +public class SearchPreferenceIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchPreferenceIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override public Settings nodeSettings(int nodeOrdinal) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java index b4d7269bab106..82dd6225fda4e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.profile.aggregation; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.InternalAggregation; @@ -46,15 +50,19 @@ import org.opensearch.search.profile.query.CollectorResult; import org.opensearch.search.profile.query.QueryProfileShardResult; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.core.IsNull; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.diversifiedSampler; import static org.opensearch.search.aggregations.AggregationBuilders.global; @@ -75,7 +83,8 @@ import static org.hamcrest.Matchers.sameInstance; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class AggregationProfilerIT extends OpenSearchIntegTestCase { +public class AggregationProfilerIT extends ParameterizedOpenSearchIntegTestCase { + private static final String BUILD_LEAF_COLLECTOR = AggregationTimingType.BUILD_LEAF_COLLECTOR.toString(); private static final String COLLECT = AggregationTimingType.COLLECT.toString(); private static final String POST_COLLECTION = AggregationTimingType.POST_COLLECTION.toString(); @@ -157,6 +166,23 @@ public class AggregationProfilerIT extends OpenSearchIntegTestCase { private static final String REASON_SEARCH_TOP_HITS = "search_top_hits"; private static final String REASON_AGGREGATION = "aggregation"; + public AggregationProfilerIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected int numberOfShards() { return 1; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/ExistsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/ExistsIT.java index 0ca1780410e13..e3253ea583ac2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/ExistsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/ExistsIT.java @@ -32,17 +32,23 @@ package org.opensearch.search.query; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.explain.ExplainResponse; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -51,11 +57,29 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -public class ExistsIT extends OpenSearchIntegTestCase { +public class ExistsIT extends ParameterizedOpenSearchIntegTestCase { + + public ExistsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } // TODO: move this to a unit test somewhere... public void testEmptyIndex() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/MultiMatchQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/MultiMatchQueryIT.java index 0c5fa4369da22..457114bac33b8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/MultiMatchQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/MultiMatchQueryIT.java @@ -31,6 +31,7 @@ package org.opensearch.search.query; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -38,6 +39,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; @@ -52,11 +54,12 @@ import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.MockKeywordPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -73,6 +76,7 @@ import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.index.query.QueryBuilders.multiMatchQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFirstHit; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -88,7 +92,24 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThan; -public class MultiMatchQueryIT extends OpenSearchIntegTestCase { +public class MultiMatchQueryIT extends ParameterizedOpenSearchIntegTestCase { + + public MultiMatchQueryIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java index 53a41af46790b..099eb934f4f4d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java @@ -32,10 +32,13 @@ package org.opensearch.search.query; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperService; @@ -44,18 +47,21 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchModule; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.Before; import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.queryStringQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.StreamsUtils.copyToStringFromClasspath; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -64,10 +70,27 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -public class QueryStringIT extends OpenSearchIntegTestCase { +public class QueryStringIT extends ParameterizedOpenSearchIntegTestCase { private static int CLUSTER_MAX_CLAUSE_COUNT; + public QueryStringIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @BeforeClass public static void createRandomClusterSetting() { CLUSTER_MAX_CLAUSE_COUNT = randomIntBetween(50, 100); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java index d736365a6e236..7ba582811bbc2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java @@ -32,10 +32,13 @@ package org.opensearch.search.query; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.RangeQueryBuilder; @@ -43,8 +46,9 @@ import org.opensearch.script.MockScriptPlugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -54,6 +58,7 @@ import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.index.query.QueryBuilders.scriptScoreQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFirstHit; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @@ -62,7 +67,24 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertThirdHit; import static org.opensearch.test.hamcrest.OpenSearchAssertions.hasScore; -public class ScriptScoreQueryIT extends OpenSearchIntegTestCase { +public class ScriptScoreQueryIT extends ParameterizedOpenSearchIntegTestCase { + + public ScriptScoreQueryIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java index 1c3a58817e48a..53bded1fc493c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.query; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.MultiTermQuery; @@ -49,6 +51,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -77,7 +80,7 @@ import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.test.InternalSettingsPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.junit.annotations.TestIssueLogging; import java.io.IOException; @@ -126,6 +129,7 @@ import static org.opensearch.index.query.QueryBuilders.termsQuery; import static org.opensearch.index.query.QueryBuilders.wildcardQuery; import static org.opensearch.index.query.QueryBuilders.wrapperQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFirstHit; @@ -143,7 +147,24 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -public class SearchQueryIT extends OpenSearchIntegTestCase { +public class SearchQueryIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchQueryIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java index bccbce3b29b8e..384d2b7423e66 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.query; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -41,6 +43,7 @@ import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -57,11 +60,12 @@ import org.opensearch.search.SearchHits; import org.opensearch.search.SearchModule; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -75,6 +79,7 @@ import static org.opensearch.index.query.QueryBuilders.queryStringQuery; import static org.opensearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.StreamsUtils.copyToStringFromClasspath; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; @@ -90,10 +95,27 @@ /** * Tests for the {@code simple_query_string} query */ -public class SimpleQueryStringIT extends OpenSearchIntegTestCase { +public class SimpleQueryStringIT extends ParameterizedOpenSearchIntegTestCase { private static int CLUSTER_MAX_CLAUSE_COUNT; + public SimpleQueryStringIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @BeforeClass public static void createRandomClusterSetting() { CLUSTER_MAX_CLAUSE_COUNT = randomIntBetween(60, 100); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/slice/SearchSliceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/slice/SearchSliceIT.java index d50f750a2b2ec..27a56f9d14f08 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/slice/SearchSliceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/slice/SearchSliceIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.slice; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.CreatePitAction; @@ -43,6 +45,7 @@ import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.Scroll; @@ -50,21 +53,41 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.sort.SortBuilders; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.concurrent.ExecutionException; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; -public class SearchSliceIT extends OpenSearchIntegTestCase { +public class SearchSliceIT extends ParameterizedOpenSearchIntegTestCase { + public SearchSliceIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private void setupIndex(int numDocs, int numberOfShards) throws IOException, ExecutionException, InterruptedException { String mapping = XContentFactory.jsonBuilder() .startObject() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java index 4f6dd89285bee..c98a38ea0bb97 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java @@ -31,26 +31,51 @@ package org.opensearch.search.source; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.join.ScoreMode; import org.opensearch.ExceptionsHelper; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.InnerHitBuilder; import org.opensearch.index.query.NestedQueryBuilder; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.search.SearchException; import org.opensearch.search.SearchHits; import org.opensearch.search.fetch.subphase.FetchSourceContext; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class MetadataFetchingIT extends OpenSearchIntegTestCase { +public class MetadataFetchingIT extends ParameterizedOpenSearchIntegTestCase { + + public MetadataFetchingIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testSimple() { assertAcked(prepareCreate("test")); ensureGreen(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java index 11223d11ff30d..eeef5403fe898 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java @@ -32,14 +32,40 @@ package org.opensearch.search.source; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.IsEqual.equalTo; -public class SourceFetchingIT extends OpenSearchIntegTestCase { +public class SourceFetchingIT extends ParameterizedOpenSearchIntegTestCase { + + public SourceFetchingIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testSourceDefaultBehavior() { createIndex("test"); ensureGreen(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java index 21453bbd17375..e0c8c93c1037a 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -32,11 +32,14 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.document.DocumentField; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -45,20 +48,24 @@ import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.geo.RandomGeoGenerator; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.SuiteScopeTestCase -public abstract class AbstractGeoTestCase extends OpenSearchIntegTestCase { +public abstract class AbstractGeoTestCase extends ParameterizedOpenSearchIntegTestCase { protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; @@ -79,6 +86,23 @@ public abstract class AbstractGeoTestCase extends OpenSearchIntegTestCase { protected static Map expectedCentroidsForGeoHash = null; protected static final double GEOHASH_TOLERANCE = 1E-5D; + public AbstractGeoTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex(UNMAPPED_IDX_NAME); From 91821c79e3d1eb562a3829818cf397e328b84344 Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Wed, 13 Sep 2023 05:54:18 -0700 Subject: [PATCH 023/388] Fix bwc on main for threadpool wait time metric (#10004) Signed-off-by: Jay Deng --- .../rest-api-spec/test/cat.thread_pool/10_basic.yml | 13 ++++++------- .../org/opensearch/threadpool/ThreadPoolStats.java | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml index 9d08f2e4ba53b..00ec838489f63 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml @@ -1,7 +1,7 @@ "Test cat thread_pool total_wait_time output": - skip: - version: " - 3.0.0" - reason: thread_pool total_wait_time stats were introduced in V_3.0.0 + version: " - 2.10.99" + reason: thread_pool total_wait_time stats were introduced in V_2.11.0 - do: cat.thread_pool: {} @@ -19,11 +19,10 @@ - match: $body: | - /^ id \s+ name \s+ total_wait_time \s+ twt \n - (\S+ \s+ search \s+ \d+s \s+ \d+ \n - \S+ \s+ search_throttled \s+ \d+s \s+ \d+ \n - \S+ \s+ index_searcher \s+ \d+s \s+ \d+ \n - \S+ \s+ generic \s+ -1 \s+ -1 \n)+ $/ + /^ name \s+ total_wait_time \s+ twt \n + (generic \s+ -1 \s+ -1 \n + search \s+ \d*\.*\d*\D+ \s+ \d*\.*\d*\D+ \n + search_throttled \s+ \d*\.*\d*\D+ \s+ \d*\.*\d*\D+ \n)+ $/ --- "Test cat thread_pool output": diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java index 8b1e1461d8b02..7b4c1504d927a 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java @@ -88,7 +88,7 @@ public Stats(StreamInput in) throws IOException { rejected = in.readLong(); largest = in.readInt(); completed = in.readLong(); - waitTimeNanos = in.getVersion().onOrAfter(Version.V_3_0_0) ? in.readLong() : -1; + waitTimeNanos = in.getVersion().onOrAfter(Version.V_2_11_0) ? in.readLong() : -1; } @Override @@ -100,7 +100,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(rejected); out.writeInt(largest); out.writeLong(completed); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_11_0)) { out.writeLong(waitTimeNanos); } } From bc404282335ed37f9f2cfc2732ca8a24cb294695 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 13 Sep 2023 13:12:45 -0400 Subject: [PATCH 024/388] Bump mockito and bytebuddy (#10022) Signed-off-by: Andriy Redko --- CHANGELOG.md | 2 ++ buildSrc/version.properties | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425177bfd2cf6..ee0b28aaeaf00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump OpenTelemetry from 1.26.0 to 1.30.1 ([#9950](https://github.com/opensearch-project/OpenSearch/pull/9950)) - Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973, #9972](https://github.com/opensearch-project/OpenSearch/pull/9973, https://github.com/opensearch-project/OpenSearch/pull/9972)) - Bump `com.google.cloud:google-cloud-core-http` from 2.21.1 to 2.23.0 ([#9971](https://github.com/opensearch-project/OpenSearch/pull/9971)) +- Bump `mockito` from 5.4.0 to 5.5.0 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022)) +- Bump `bytebuddy` from 1.14.3 to 1.14.7 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index ee666f90f83dd..be09584b1dba1 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -53,9 +53,9 @@ bouncycastle=1.75 randomizedrunner = 2.7.1 junit = 4.13.2 hamcrest = 2.1 -mockito = 5.4.0 +mockito = 5.5.0 objenesis = 3.2 -bytebuddy = 1.14.3 +bytebuddy = 1.14.7 # benchmark dependencies jmh = 1.35 From 1eb8b3a5a9c147d0c88f95a398452167c8fc69ed Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Wed, 13 Sep 2023 14:40:54 -0700 Subject: [PATCH 025/388] Add workflow to copy labels from linked issues to the PR (#10010) Signed-off-by: Kunal Kotwani --- .../workflows/copy-linked-issue-labels.yml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/copy-linked-issue-labels.yml diff --git a/.github/workflows/copy-linked-issue-labels.yml b/.github/workflows/copy-linked-issue-labels.yml new file mode 100644 index 0000000000000..33b5e92dc10da --- /dev/null +++ b/.github/workflows/copy-linked-issue-labels.yml @@ -0,0 +1,21 @@ +name: Copy labels from linked issues +on: + pull_request_target: + types: [opened, edited, review_requested, synchronize, reopened, ready_for_review] + +jobs: + copy-issue-labels: + if: github.repository == 'opensearch-project/OpenSearch' + runs-on: ubuntu-latest + permissions: + issues: read + contents: read + pull-requests: write + steps: + - name: copy-issue-labels + uses: michalvankodev/copy-issue-labels@v1.3.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + labels-to-exclude: | + untriaged + triaged From 8b1ab232796e4f8d226afa2354f79b832b9626ee Mon Sep 17 00:00:00 2001 From: Neetika Singhal Date: Wed, 13 Sep 2023 15:26:11 -0700 Subject: [PATCH 026/388] Parameterize tests for concurrent search 2 (#9852) Signed-off-by: Neetika Singhal --- .../AggregationsIntegrationIT.java | 27 +++++++++++++++- .../search/aggregations/CombiIT.java | 28 +++++++++++++++-- .../search/aggregations/MetadataIT.java | 28 +++++++++++++++-- .../aggregations/bucket/BooleanTermsIT.java | 28 ++++++++++++++++- .../aggregations/bucket/DateRangeIT.java | 27 +++++++++++++++- .../bucket/DiversifiedSamplerIT.java | 25 ++++++++++++++- .../aggregations/bucket/DoubleTermsIT.java | 5 +++ .../search/aggregations/bucket/FilterIT.java | 27 +++++++++++++++- .../search/aggregations/bucket/FiltersIT.java | 25 ++++++++++++++- .../aggregations/bucket/GeoDistanceIT.java | 25 ++++++++++++++- .../search/aggregations/bucket/GlobalIT.java | 27 +++++++++++++++- .../aggregations/bucket/HistogramIT.java | 28 ++++++++++++++++- .../search/aggregations/bucket/IpRangeIT.java | 25 ++++++++++++++- .../search/aggregations/bucket/IpTermsIT.java | 5 +++ .../aggregations/bucket/LongTermsIT.java | 5 +++ .../aggregations/bucket/MinDocCountIT.java | 5 +++ .../aggregations/bucket/MultiTermsIT.java | 5 +++ .../aggregations/bucket/NaNSortingIT.java | 28 ++++++++++++++++- .../search/aggregations/bucket/NestedIT.java | 31 ++++++++++++++++++- .../search/aggregations/bucket/RangeIT.java | 26 +++++++++++++++- .../aggregations/bucket/ReverseNestedIT.java | 26 +++++++++++++++- .../search/aggregations/bucket/SamplerIT.java | 26 +++++++++++++++- .../aggregations/bucket/ShardReduceIT.java | 28 ++++++++++++++++- .../SignificantTermsSignificanceScoreIT.java | 24 +++++++++++++- .../bucket/TermsDocCountErrorIT.java | 26 +++++++++++++++- .../bucket/terms/BaseStringTermsTestCase.java | 5 +++ .../bucket/terms/StringTermsIT.java | 5 +++ .../aggregations/metrics/CardinalityIT.java | 26 +++++++++++++++- .../aggregations/metrics/ExtendedStatsIT.java | 5 +++ .../metrics/HDRPercentileRanksIT.java | 5 +++ .../metrics/HDRPercentilesIT.java | 5 +++ .../metrics/MedianAbsoluteDeviationIT.java | 5 +++ .../metrics/ScriptedMetricIT.java | 26 +++++++++++++++- .../search/aggregations/metrics/StatsIT.java | 5 +++ .../search/aggregations/metrics/SumIT.java | 5 +++ .../metrics/TDigestPercentileRanksIT.java | 5 +++ .../metrics/TDigestPercentilesIT.java | 5 +++ .../aggregations/metrics/TopHitsIT.java | 25 ++++++++++++++- .../aggregations/metrics/ValueCountIT.java | 27 +++++++++++++++- .../aggregations/pipeline/AvgBucketIT.java | 27 +++++++++++++++- .../aggregations/pipeline/BucketScriptIT.java | 26 +++++++++++++++- .../pipeline/BucketSelectorIT.java | 26 +++++++++++++++- .../aggregations/pipeline/BucketSortIT.java | 26 +++++++++++++++- .../pipeline/DateDerivativeIT.java | 26 +++++++++++++++- .../aggregations/pipeline/DerivativeIT.java | 28 ++++++++++++++++- .../pipeline/ExtendedStatsBucketIT.java | 27 +++++++++++++++- .../aggregations/pipeline/MaxBucketIT.java | 28 ++++++++++++++++- .../aggregations/pipeline/MinBucketIT.java | 27 +++++++++++++++- .../aggregations/pipeline/MovAvgIT.java | 26 +++++++++++++++- .../pipeline/PercentilesBucketIT.java | 26 +++++++++++++++- .../aggregations/pipeline/SerialDiffIT.java | 27 +++++++++++++++- .../aggregations/pipeline/StatsBucketIT.java | 28 +++++++++++++++-- .../aggregations/pipeline/SumBucketIT.java | 27 +++++++++++++++- .../opensearch/search/sort/GeoDistanceIT.java | 27 ++++++++++++++-- .../bucket/AbstractTermsTestCase.java | 29 +++++++++++++++-- .../metrics/AbstractNumericTestCase.java | 27 +++++++++++++++- 56 files changed, 1126 insertions(+), 46 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java index b73b7722f9728..257786c1e9ce5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java @@ -32,11 +32,15 @@ package org.opensearch.search.aggregations; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.bucket.terms.IncludeExclude; import org.opensearch.search.aggregations.bucket.terms.RareTermsAggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.SignificantTermsAggregationBuilder; @@ -45,16 +49,20 @@ import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.TermsAggregatorFactory; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class AggregationsIntegrationIT extends OpenSearchIntegTestCase { +public class AggregationsIntegrationIT extends ParameterizedOpenSearchIntegTestCase { static int numDocs; @@ -63,6 +71,23 @@ public class AggregationsIntegrationIT extends OpenSearchIntegTestCase { + LARGE_STRING.length() + "] used in the request has exceeded the allowed maximum"; + public AggregationsIntegrationIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(prepareCreate("index").setMapping("f", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java index d35a560b0986c..3d3cf1943dfe3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java @@ -32,20 +32,27 @@ package org.opensearch.search.aggregations; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.missing.Missing; import org.opensearch.search.aggregations.bucket.terms.Terms; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.missing; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -54,7 +61,24 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNull.notNullValue; -public class CombiIT extends OpenSearchIntegTestCase { +public class CombiIT extends ParameterizedOpenSearchIntegTestCase { + + public CombiIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } /** * Making sure that if there are multiple aggregations, working on the same field, yet require different diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java index f210af7c10fb3..1bc0cb36f5fe3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java @@ -32,25 +32,49 @@ package org.opensearch.search.aggregations; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.InternalBucketMetricValue; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.maxBucket; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -public class MetadataIT extends OpenSearchIntegTestCase { +public class MetadataIT extends ParameterizedOpenSearchIntegTestCase { + + public MetadataIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public void testMetadataSetOnAggregationResult() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("name", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/BooleanTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/BooleanTermsIT.java index fc5407c4cade8..7ab1a44ce220c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/BooleanTermsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/BooleanTermsIT.java @@ -31,26 +31,52 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class BooleanTermsIT extends OpenSearchIntegTestCase { +public class BooleanTermsIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "b_value"; private static final String MULTI_VALUED_FIELD_NAME = "b_values"; static int numSingleTrues, numSingleFalses, numMultiTrues, numMultiFalses; + public BooleanTermsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateRangeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateRangeIT.java index 470ee6a4d2cea..ae4243019ffb1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateRangeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateRangeIT.java @@ -31,11 +31,14 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.plugins.Plugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; @@ -46,6 +49,7 @@ import org.opensearch.search.aggregations.bucket.range.Range.Bucket; import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.time.ZoneId; @@ -63,6 +67,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; @@ -76,7 +81,24 @@ import static org.hamcrest.core.IsNull.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class DateRangeIT extends OpenSearchIntegTestCase { +public class DateRangeIT extends ParameterizedOpenSearchIntegTestCase { + + public DateRangeIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } private static IndexRequestBuilder indexDoc(int month, int day, int value) throws Exception { return client().prepareIndex("idx") @@ -1062,6 +1084,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } /** @@ -1124,6 +1147,7 @@ public void testRangeWithFormatStringValue() throws Exception { .get() ); assertThat(e.getDetailedMessage(), containsString("failed to parse date field [1000000] with format [strict_hour_minute_second]")); + internalCluster().wipeIndices(indexName); } /** @@ -1196,6 +1220,7 @@ public void testRangeWithFormatNumericValue() throws Exception { buckets = checkBuckets(searchResponse.getAggregations().get("date_range"), "date_range", 2); assertBucket(buckets.get(0), 2L, "1000000-3000000", 1000000L, 3000000L); assertBucket(buckets.get(1), 1L, "3000000-4000000", 3000000L, 4000000L); + internalCluster().wipeIndices(indexName); } private static List checkBuckets(Range dateRange, String expectedAggName, long expectedBucketsSize) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java index 0d133a933df1f..5e95073209c71 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java @@ -31,10 +31,13 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder; @@ -45,12 +48,15 @@ import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.aggregations.metrics.Max; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; import java.util.Collection; import java.util.List; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.max; import static org.opensearch.search.aggregations.AggregationBuilders.sampler; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -65,10 +71,27 @@ * Tests the Sampler aggregation */ @OpenSearchIntegTestCase.SuiteScopeTestCase -public class DiversifiedSamplerIT extends OpenSearchIntegTestCase { +public class DiversifiedSamplerIT extends ParameterizedOpenSearchIntegTestCase { public static final int NUM_SHARDS = 2; + public DiversifiedSamplerIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public String randomExecutionHint() { return randomBoolean() ? null : randomFrom(SamplerAggregator.ExecutionMode.values()).toString(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DoubleTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DoubleTermsIT.java index b740271cdef77..88bb41923e53f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DoubleTermsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DoubleTermsIT.java @@ -88,6 +88,10 @@ @OpenSearchIntegTestCase.SuiteScopeTestCase public class DoubleTermsIT extends AbstractTermsTestCase { + public DoubleTermsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); @@ -1106,5 +1110,6 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterIT.java index ef455bf353ce4..7aa98803403e0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterIT.java @@ -31,9 +31,13 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; @@ -42,14 +46,18 @@ import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.metrics.Avg; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -60,10 +68,27 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class FilterIT extends OpenSearchIntegTestCase { +public class FilterIT extends ParameterizedOpenSearchIntegTestCase { static int numDocs, numTag1Docs; + public FilterIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FiltersIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FiltersIT.java index 4c5033b957d00..b6cf515df78ba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FiltersIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FiltersIT.java @@ -32,9 +32,13 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; @@ -44,6 +48,7 @@ import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.metrics.Avg; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; @@ -56,6 +61,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.filters; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -66,10 +72,27 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class FiltersIT extends OpenSearchIntegTestCase { +public class FiltersIT extends ParameterizedOpenSearchIntegTestCase { static int numDocs, numTag1Docs, numTag2Docs, numOtherDocs; + public FiltersIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoDistanceIT.java index 6d99424989fd7..025bebf8b254d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoDistanceIT.java @@ -31,6 +31,8 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.Version; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; @@ -39,6 +41,7 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.DistanceUnit; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.InternalAggregation; @@ -47,17 +50,20 @@ import org.opensearch.search.aggregations.bucket.range.Range.Bucket; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; import org.hamcrest.Matchers; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.geoDistance; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -70,7 +76,11 @@ import static org.hamcrest.core.IsNull.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoDistanceIT extends OpenSearchIntegTestCase { +public class GeoDistanceIT extends ParameterizedOpenSearchIntegTestCase { + + public GeoDistanceIT(Settings dynamicSettings) { + super(dynamicSettings); + } @Override protected boolean forbidPrivateIndexSettings() { @@ -79,6 +89,19 @@ protected boolean forbidPrivateIndexSettings() { private Version version = VersionUtils.randomIndexCompatibleVersion(random()); + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private IndexRequestBuilder indexCity(String idx, String name, String... latLons) throws Exception { XContentBuilder source = jsonBuilder().startObject().field("city", name); source.startArray("location"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GlobalIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GlobalIT.java index 8a97d9c9e75dd..be31a3afadad0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GlobalIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GlobalIT.java @@ -31,19 +31,27 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.metrics.Stats; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.stats; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; @@ -53,10 +61,27 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GlobalIT extends OpenSearchIntegTestCase { +public class GlobalIT extends ParameterizedOpenSearchIntegTestCase { static int numDocs; + public GlobalIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java index 6d5918ffa7f0d..75f57d1cc4c0e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/HistogramIT.java @@ -31,11 +31,14 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.Plugin; import org.opensearch.script.MockScriptPlugin; @@ -53,10 +56,12 @@ import org.opensearch.search.aggregations.metrics.Stats; import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -69,6 +74,7 @@ import static java.util.Collections.emptyMap; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -85,7 +91,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class HistogramIT extends OpenSearchIntegTestCase { +public class HistogramIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; private static final String MULTI_VALUED_FIELD_NAME = "l_values"; @@ -96,6 +102,23 @@ public class HistogramIT extends OpenSearchIntegTestCase { static long[] valueCounts, valuesCounts; static Map> expectedMultiSortBuckets; + public HistogramIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); @@ -1144,6 +1167,7 @@ public void testDecimalIntervalAndOffset() throws Exception { assertEquals(1, buckets.get(0).getDocCount()); assertEquals(0.05, (double) buckets.get(1).getKey(), 0.01d); assertEquals(1, buckets.get(1).getDocCount()); + internalCluster().wipeIndices("decimal_values"); } /** @@ -1285,6 +1309,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndKeyDesc() throws Exception { @@ -1388,6 +1413,7 @@ public void testHardBounds() throws Exception { buckets = histogram.getBuckets(); assertEquals(1, buckets.size()); assertEquals(0.1, (double) buckets.get(0).getKey(), 0.01d); + internalCluster().wipeIndices("test"); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpRangeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpRangeIT.java index f8f666aaa3c1b..14a3685bd183e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpRangeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpRangeIT.java @@ -31,9 +31,13 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.plugins.Plugin; import org.opensearch.script.MockScriptPlugin; import org.opensearch.script.Script; @@ -41,6 +45,7 @@ import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.bucket.range.Range; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.Arrays; import java.util.Collection; @@ -48,13 +53,31 @@ import java.util.Map; import java.util.function.Function; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class IpRangeIT extends OpenSearchIntegTestCase { +public class IpRangeIT extends ParameterizedOpenSearchIntegTestCase { + + public IpRangeIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public static class DummyScriptPlugin extends MockScriptPlugin { @Override diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpTermsIT.java index cff51e74fdbd0..c712c97af5c71 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpTermsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/IpTermsIT.java @@ -32,6 +32,7 @@ package org.opensearch.search.aggregations.bucket; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.plugins.Plugin; import org.opensearch.script.Script; @@ -50,6 +51,10 @@ public class IpTermsIT extends AbstractTermsTestCase { + public IpTermsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/LongTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/LongTermsIT.java index f3d77ac1236be..345cbdae8ef07 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/LongTermsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/LongTermsIT.java @@ -86,6 +86,10 @@ @OpenSearchIntegTestCase.SuiteScopeTestCase public class LongTermsIT extends AbstractTermsTestCase { + public LongTermsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); @@ -1054,5 +1058,6 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java index 4c5d9fb60d4f7..90dafc0d57887 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java @@ -37,6 +37,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.query.QueryBuilder; @@ -81,6 +82,10 @@ public class MinDocCountIT extends AbstractTermsTestCase { private static final QueryBuilder QUERY = QueryBuilders.termQuery("match", true); private static int cardinality; + public MinDocCountIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MultiTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MultiTermsIT.java index 950f7560dfea3..ea5a59d89309f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MultiTermsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MultiTermsIT.java @@ -9,6 +9,7 @@ package org.opensearch.search.aggregations.bucket; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.bucket.terms.BaseStringTermsTestCase; @@ -33,6 +34,10 @@ @OpenSearchIntegTestCase.SuiteScopeTestCase public class MultiTermsIT extends BaseStringTermsTestCase { + public MultiTermsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard public void testSizeIsZero() { final int minDocCount = randomInt(1); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java index 3b3f169f7578b..1ef2c0e8db8c7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.Comparators; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -47,8 +51,13 @@ import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.extendedStats; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -58,7 +67,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class NaNSortingIT extends OpenSearchIntegTestCase { +public class NaNSortingIT extends ParameterizedOpenSearchIntegTestCase { private enum SubAggregation { AVG("avg") { @@ -130,6 +139,23 @@ public String sortKey() { public abstract double getValue(Aggregation aggregation); } + public NaNSortingIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("string_value", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java index 7efb16c8b719c..b3009ffcf4f45 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java @@ -31,12 +31,15 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.join.ScoreMode; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -54,9 +57,12 @@ import org.opensearch.search.aggregations.metrics.Stats; import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; @@ -66,6 +72,7 @@ import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.nestedQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.max; @@ -85,12 +92,29 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class NestedIT extends OpenSearchIntegTestCase { +public class NestedIT extends ParameterizedOpenSearchIntegTestCase { private static int numParents; private static int[] numChildren; private static SubAggCollectionMode aggCollectionMode; + public NestedIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { @@ -532,6 +556,7 @@ public void testParentFilterResolvedCorrectly() throws Exception { assertThat(nestedTags.getDocCount(), equalTo(0L)); // This must be 0 tags = nestedTags.getAggregations().get("tag"); assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty + internalCluster().wipeIndices("idx2"); } public void testNestedSameDocIdProcessedMultipleTime() throws Exception { @@ -638,6 +663,7 @@ public void testNestedSameDocIdProcessedMultipleTime() throws Exception { assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(1L)); assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L)); assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L)); + internalCluster().wipeIndices("idx4"); } public void testFilterAggInsideNestedAgg() throws Exception { @@ -800,6 +826,7 @@ public void testFilterAggInsideNestedAgg() throws Exception { assertThat(bucket.getDocCount(), equalTo(1L)); numStringParams = bucket.getAggregations().get("num_string_params"); assertThat(numStringParams.getDocCount(), equalTo(0L)); + internalCluster().wipeIndices("classes"); } public void testExtractInnerHitBuildersWithDuplicateHitName() throws Exception { @@ -824,6 +851,7 @@ public void testExtractInnerHitBuildersWithDuplicateHitName() throws Exception { RestStatus.BAD_REQUEST, containsString("[inner_hits] already contains an entry for key [ih1]") ); + internalCluster().wipeIndices("idxduplicatehitnames"); } public void testExtractInnerHitBuildersWithDuplicatePath() throws Exception { @@ -846,5 +874,6 @@ public void testExtractInnerHitBuildersWithDuplicatePath() throws Exception { RestStatus.BAD_REQUEST, containsString("[inner_hits] already contains an entry for key [property]") ); + internalCluster().wipeIndices("idxnullhitnames"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java index c46d6dcd847e1..64ab6f1382ac3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java @@ -31,10 +31,13 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.plugins.Plugin; import org.opensearch.script.Script; @@ -48,9 +51,11 @@ import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -60,6 +65,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.range; import static org.opensearch.search.aggregations.AggregationBuilders.sum; @@ -73,13 +79,30 @@ import static org.hamcrest.core.IsNull.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class RangeIT extends OpenSearchIntegTestCase { +public class RangeIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; private static final String MULTI_VALUED_FIELD_NAME = "l_values"; static int numDocs; + public RangeIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); @@ -1061,6 +1084,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } public void testFieldAlias() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ReverseNestedIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ReverseNestedIT.java index 749f2170dab50..2716db6b7e745 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ReverseNestedIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ReverseNestedIT.java @@ -31,9 +31,12 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.BucketOrder; @@ -44,9 +47,11 @@ import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.metrics.ValueCount; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -55,6 +60,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.count; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.nested; @@ -70,7 +76,24 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class ReverseNestedIT extends OpenSearchIntegTestCase { +public class ReverseNestedIT extends ParameterizedOpenSearchIntegTestCase { + + public ReverseNestedIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override public void setupSuiteScopeCluster() throws Exception { @@ -726,6 +749,7 @@ public void testSameParentDocHavingMultipleBuckets() throws Exception { ValueCount barCount = reverseToBar.getAggregations().get("sku_count"); assertThat(barCount.getValue(), equalTo(2L)); } + internalCluster().wipeIndices("idx3"); } public void testFieldAlias() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java index 587bf2a707710..7033c42c5d661 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java @@ -32,10 +32,13 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.sampler.Sampler; @@ -45,11 +48,15 @@ import org.opensearch.search.aggregations.bucket.terms.Terms.Bucket; import org.opensearch.search.aggregations.metrics.Max; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.max; import static org.opensearch.search.aggregations.AggregationBuilders.sampler; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -64,7 +71,7 @@ * Tests the Sampler aggregation */ @OpenSearchIntegTestCase.SuiteScopeTestCase -public class SamplerIT extends OpenSearchIntegTestCase { +public class SamplerIT extends ParameterizedOpenSearchIntegTestCase { public static final int NUM_SHARDS = 2; @@ -72,6 +79,23 @@ public String randomExecutionHint() { return randomBoolean() ? null : randomFrom(SamplerAggregator.ExecutionMode.values()).toString(); } + public SamplerIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java index faa6a54394b00..66d761c56634e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java @@ -31,8 +31,12 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.geometry.utils.Geohash; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -45,8 +49,13 @@ import org.opensearch.search.aggregations.bucket.range.Range; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.filter; @@ -68,7 +77,24 @@ * we can make sure that the reduce is properly propagated by checking that empty buckets were created. */ @OpenSearchIntegTestCase.SuiteScopeTestCase -public class ShardReduceIT extends OpenSearchIntegTestCase { +public class ShardReduceIT extends ParameterizedOpenSearchIntegTestCase { + + public ShardReduceIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } private IndexRequestBuilder indexDoc(String date, int value) throws Exception { return client().prepareIndex("idx") diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index 5bf403d19ed9f..e914b87754865 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -31,11 +31,14 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -62,6 +65,7 @@ import org.opensearch.search.aggregations.bucket.terms.heuristic.ScriptHeuristic; import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.search.aggregations.bucket.SharedSignificantTermsTestMethods; import java.io.IOException; @@ -78,6 +82,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.significantTerms; import static org.opensearch.search.aggregations.AggregationBuilders.significantText; @@ -90,12 +95,29 @@ import static org.hamcrest.Matchers.is; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class SignificantTermsSignificanceScoreIT extends OpenSearchIntegTestCase { +public class SignificantTermsSignificanceScoreIT extends ParameterizedOpenSearchIntegTestCase { static final String INDEX_NAME = "testidx"; static final String TEXT_FIELD = "text"; static final String CLASS_FIELD = "class"; + public SignificantTermsSignificanceScoreIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Arrays.asList(TestScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java index 63385b55f47e8..3fcf4b5d533d4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java @@ -32,23 +32,30 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.bucket.terms.Terms.Bucket; import org.opensearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -60,7 +67,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class TermsDocCountErrorIT extends OpenSearchIntegTestCase { +public class TermsDocCountErrorIT extends ParameterizedOpenSearchIntegTestCase { private static final String STRING_FIELD_NAME = "s_value"; private static final String LONG_FIELD_NAME = "l_value"; @@ -72,6 +79,23 @@ public static String randomExecutionHint() { private static int numRoutingValues; + public TermsDocCountErrorIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping(STRING_FIELD_NAME, "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/BaseStringTermsTestCase.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/BaseStringTermsTestCase.java index 969cbf272fab0..20caa4fd076fe 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/BaseStringTermsTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/BaseStringTermsTestCase.java @@ -9,6 +9,7 @@ package org.opensearch.search.aggregations.bucket.terms; import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.plugins.Plugin; @@ -37,6 +38,10 @@ public class BaseStringTermsTestCase extends AbstractTermsTestCase { protected static final String MULTI_VALUED_FIELD_NAME = "s_values"; protected static Map> expectedMultiSortBuckets; + public BaseStringTermsTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/StringTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/StringTermsIT.java index 1f1da9627d5ea..8c727d280ec52 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/StringTermsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/terms/StringTermsIT.java @@ -79,6 +79,10 @@ @OpenSearchIntegTestCase.SuiteScopeTestCase public class StringTermsIT extends BaseStringTermsTestCase { + public StringTermsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard public void testSizeIsZero() { final int minDocCount = randomInt(1); @@ -1127,6 +1131,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } public void testScriptWithValueType() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java index 147f451c14de8..9ebec21367164 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java @@ -32,9 +32,12 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.plugins.Plugin; import org.opensearch.script.MockScriptPlugin; @@ -45,7 +48,9 @@ import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -55,6 +60,7 @@ import static java.util.Collections.emptyMap; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.cardinality; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -65,7 +71,24 @@ import static org.hamcrest.Matchers.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class CardinalityIT extends OpenSearchIntegTestCase { +public class CardinalityIT extends ParameterizedOpenSearchIntegTestCase { + + public CardinalityIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { @@ -615,5 +638,6 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ExtendedStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ExtendedStatsIT.java index 2efb49c488d76..3d804b9aa626e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ExtendedStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ExtendedStatsIT.java @@ -70,6 +70,10 @@ public class ExtendedStatsIT extends AbstractNumericTestCase { + public ExtendedStatsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(AggregationTestScriptsPlugin.class); @@ -995,6 +999,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentileRanksIT.java index 87e5a73ef630d..7ca5130388eea 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentileRanksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentileRanksIT.java @@ -72,6 +72,10 @@ public class HDRPercentileRanksIT extends AbstractNumericTestCase { + public HDRPercentileRanksIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(AggregationTestScriptsPlugin.class); @@ -716,6 +720,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentilesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentilesIT.java index ad3fd6517d1b1..ec913b3e130f5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentilesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/HDRPercentilesIT.java @@ -75,6 +75,10 @@ public class HDRPercentilesIT extends AbstractNumericTestCase { + public HDRPercentilesIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(AggregationTestScriptsPlugin.class); @@ -687,6 +691,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java index 6af65beba6124..b8447d682abae 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java @@ -91,6 +91,10 @@ public class MedianAbsoluteDeviationIT extends AbstractNumericTestCase { private static double singleValueExactMAD; private static double multiValueExactMAD; + public MedianAbsoluteDeviationIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override public void setupSuiteScopeCluster() throws Exception { final Settings settings = Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build(); @@ -643,5 +647,6 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java index 5c782c6d085b4..ced2358ac3f78 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -32,11 +32,14 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -53,12 +56,14 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.Before; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -69,6 +74,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.scriptedMetric; @@ -87,10 +93,27 @@ @ClusterScope(scope = Scope.SUITE) @OpenSearchIntegTestCase.SuiteScopeTestCase -public class ScriptedMetricIT extends OpenSearchIntegTestCase { +public class ScriptedMetricIT extends ParameterizedOpenSearchIntegTestCase { private static long numDocs; + public ScriptedMetricIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); @@ -1378,6 +1401,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } public void testConflictingAggAndScriptParams() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java index e02657670b943..f957a74eeb9d0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/StatsIT.java @@ -66,6 +66,10 @@ import static org.hamcrest.Matchers.sameInstance; public class StatsIT extends AbstractNumericTestCase { + public StatsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(AggregationTestScriptsPlugin.class); @@ -384,5 +388,6 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java index fe236f04c19e8..382d656448114 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/SumIT.java @@ -68,6 +68,10 @@ public class SumIT extends AbstractNumericTestCase { + public SumIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(MetricAggScriptPlugin.class); @@ -359,6 +363,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } public void testFieldAlias() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentileRanksIT.java index ab0cdbaf3047f..941d3a888db29 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentileRanksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentileRanksIT.java @@ -72,6 +72,10 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase { + public TDigestPercentileRanksIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(AggregationTestScriptsPlugin.class); @@ -626,6 +630,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentilesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentilesIT.java index 2c05ed0bac44a..6457cf9307fa1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentilesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TDigestPercentilesIT.java @@ -74,6 +74,10 @@ public class TDigestPercentilesIT extends AbstractNumericTestCase { + public TDigestPercentilesIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(AggregationTestScriptsPlugin.class); @@ -597,5 +601,6 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java index 96aeccfc03fb1..10e51079cf389 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java @@ -31,6 +31,8 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.Explanation; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.util.ArrayUtil; @@ -40,6 +42,7 @@ import org.opensearch.action.search.SearchType; import org.opensearch.common.document.DocumentField; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.query.MatchAllQueryBuilder; @@ -67,8 +70,10 @@ import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -83,6 +88,7 @@ import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.index.query.QueryBuilders.nestedQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.max; @@ -105,11 +111,28 @@ import static org.hamcrest.Matchers.sameInstance; @OpenSearchIntegTestCase.SuiteScopeTestCase() -public class TopHitsIT extends OpenSearchIntegTestCase { +public class TopHitsIT extends ParameterizedOpenSearchIntegTestCase { private static final String TERMS_AGGS_FIELD = "terms"; private static final String SORT_FIELD = "sort"; + public TopHitsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java index 82e667bccc576..833d1ce3bb4c3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/ValueCountIT.java @@ -31,8 +31,11 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.plugins.Plugin; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; @@ -42,7 +45,9 @@ import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -51,6 +56,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.count; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.global; @@ -67,7 +73,25 @@ import static org.hamcrest.Matchers.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class ValueCountIT extends OpenSearchIntegTestCase { +public class ValueCountIT extends ParameterizedOpenSearchIntegTestCase { + + public ValueCountIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -363,6 +387,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + internalCluster().wipeIndices("cache_test_idx"); } public void testOrderByEmptyAggregation() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java index 6cd16a47e98d2..bec9203384026 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; @@ -42,11 +46,15 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -58,7 +66,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class AvgBucketIT extends OpenSearchIntegTestCase { +public class AvgBucketIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; @@ -69,6 +77,23 @@ public class AvgBucketIT extends OpenSearchIntegTestCase { static int numValueBuckets; static long[] valueCounts; + public AvgBucketIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("tag", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java index 926c708e99bd6..4c3129eb89e3b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -47,9 +51,11 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -58,6 +64,7 @@ import java.util.function.Function; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; @@ -69,7 +76,7 @@ import static org.hamcrest.Matchers.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class BucketScriptIT extends OpenSearchIntegTestCase { +public class BucketScriptIT extends ParameterizedOpenSearchIntegTestCase { private static final String FIELD_1_NAME = "field1"; private static final String FIELD_2_NAME = "field2"; @@ -83,6 +90,23 @@ public class BucketScriptIT extends OpenSearchIntegTestCase { private static int maxNumber; private static long date; + public BucketScriptIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java index 7b802478a46d8..a7b28add7373a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -46,9 +50,11 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -57,6 +63,7 @@ import java.util.function.Function; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.bucketSelector; @@ -70,7 +77,7 @@ import static org.hamcrest.Matchers.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class BucketSelectorIT extends OpenSearchIntegTestCase { +public class BucketSelectorIT extends ParameterizedOpenSearchIntegTestCase { private static final String FIELD_1_NAME = "field1"; private static final String FIELD_2_NAME = "field2"; @@ -82,6 +89,23 @@ public class BucketSelectorIT extends OpenSearchIntegTestCase { private static int minNumber; private static int maxNumber; + public BucketSelectorIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(CustomScriptPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java index 231aa2e078de6..2e4fd7a412118 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java @@ -32,10 +32,14 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.bucket.histogram.Histogram; @@ -44,15 +48,18 @@ import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -68,7 +75,7 @@ import static org.hamcrest.Matchers.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class BucketSortIT extends OpenSearchIntegTestCase { +public class BucketSortIT extends ParameterizedOpenSearchIntegTestCase { private static final String INDEX = "bucket-sort-it-data-index"; private static final String INDEX_WITH_GAPS = "bucket-sort-it-data-index-with-gaps"; @@ -78,6 +85,23 @@ public class BucketSortIT extends OpenSearchIntegTestCase { private static final String VALUE_1_FIELD = "value_1"; private static final String VALUE_2_FIELD = "value_2"; + public BucketSortIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex(INDEX, INDEX_WITH_GAPS); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java index 2c7890fb7b1cb..b05ff7b4329cd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java @@ -32,10 +32,14 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.InternalMultiBucketAggregation; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; @@ -44,6 +48,7 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.support.AggregationPath; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matcher; import org.junit.After; @@ -55,9 +60,11 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.derivative; @@ -69,13 +76,30 @@ import static org.hamcrest.core.IsNull.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class DateDerivativeIT extends OpenSearchIntegTestCase { +public class DateDerivativeIT extends ParameterizedOpenSearchIntegTestCase { // some index names used during these tests private static final String IDX_DST_START = "idx_dst_start"; private static final String IDX_DST_END = "idx_dst_end"; private static final String IDX_DST_KATHMANDU = "idx_dst_kathmandu"; + public DateDerivativeIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private ZonedDateTime date(int month, int day) { return ZonedDateTime.of(2012, month, day, 0, 0, 0, 0, ZoneOffset.UTC); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java index 5cff68001c8d5..18484c8a60ed7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java @@ -32,10 +32,14 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.InternalAggregation; @@ -47,14 +51,18 @@ import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.search.aggregations.support.AggregationPath; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.filters; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -70,7 +78,7 @@ import static org.hamcrest.core.IsNull.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class DerivativeIT extends OpenSearchIntegTestCase { +public class DerivativeIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; @@ -92,6 +100,23 @@ public class DerivativeIT extends OpenSearchIntegTestCase { private static Double[] firstDerivValueCounts_empty_rnd; private static long numDocsEmptyIdx_rnd; + public DerivativeIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); @@ -680,6 +705,7 @@ public void testAvgMovavgDerivNPE() throws Exception { .get(); assertSearchResponse(response); + internalCluster().wipeIndices("movavg_npe"); } private void checkBucketKeyAndDocCount( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java index 85fe794b05fc6..299827e2413d4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java @@ -32,10 +32,14 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; @@ -45,11 +49,15 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -61,7 +69,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class ExtendedStatsBucketIT extends OpenSearchIntegTestCase { +public class ExtendedStatsBucketIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; @@ -72,6 +80,23 @@ public class ExtendedStatsBucketIT extends OpenSearchIntegTestCase { static int numValueBuckets; static long[] valueCounts; + public ExtendedStatsBucketIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("tag", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java index a114fa4079e21..bb90c1294ecb8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java @@ -32,9 +32,13 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -54,12 +58,16 @@ import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; @@ -72,7 +80,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class MaxBucketIT extends OpenSearchIntegTestCase { +public class MaxBucketIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; @@ -83,6 +91,23 @@ public class MaxBucketIT extends OpenSearchIntegTestCase { static int numValueBuckets; static long[] valueCounts; + public MaxBucketIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("tag", "type=keyword").get()); @@ -587,5 +612,6 @@ public void testFieldIsntWrittenOutTwice() throws Exception { SearchResponse response = client().prepareSearch("foo_*").setSize(0).addAggregation(groupByLicenseAgg).get(); BytesReference bytes = org.opensearch.core.xcontent.XContentHelper.toXContent(response, MediaTypeRegistry.JSON, false); XContentHelper.convertToMap(bytes, false, MediaTypeRegistry.JSON); + internalCluster().wipeIndices("foo_*"); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java index a29bfc0eaa7cb..189bfd9b5b80a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; @@ -42,11 +46,15 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -58,7 +66,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class MinBucketIT extends OpenSearchIntegTestCase { +public class MinBucketIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; @@ -69,6 +77,23 @@ public class MinBucketIT extends OpenSearchIntegTestCase { static int numValueBuckets; static long[] valueCounts; + public MinBucketIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("tag", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java index b53183a627ecc..0cf89778c6e99 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.bulk.BulkRequestBuilder; import org.opensearch.action.index.IndexRequestBuilder; @@ -40,12 +42,15 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; import org.opensearch.common.collect.EvictingQueue; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.opensearch.search.aggregations.metrics.Avg; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; @@ -57,6 +62,7 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.max; @@ -71,7 +77,7 @@ import static org.hamcrest.core.IsNull.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class MovAvgIT extends OpenSearchIntegTestCase { +public class MovAvgIT extends ParameterizedOpenSearchIntegTestCase { private static final String INTERVAL_FIELD = "l_value"; private static final String VALUE_FIELD = "v_value"; private static final String VALUE_FIELD2 = "v_value2"; @@ -127,6 +133,23 @@ public String toString() { } } + public MovAvgIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { prepareCreate("idx").setMapping( @@ -1341,6 +1364,7 @@ public void testPredictWithNonEmptyBuckets() throws Exception { assertThat(movAvgAgg, nullValue()); } } + internalCluster().wipeIndices("predict_non_empty"); } private void assertValidIterators(Iterator expectedBucketIter, Iterator expectedCountsIter, Iterator expectedValuesIter) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java index 1da079781dc63..580497715ed6d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java @@ -32,10 +32,14 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.terms.IncludeExclude; @@ -43,14 +47,17 @@ import org.opensearch.search.aggregations.metrics.Percentile; import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -62,7 +69,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class PercentilesBucketIT extends OpenSearchIntegTestCase { +public class PercentilesBucketIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; private static final double[] PERCENTS = { 0.0, 1.0, 25.0, 50.0, 75.0, 99.0, 100.0 }; @@ -73,6 +80,23 @@ public class PercentilesBucketIT extends OpenSearchIntegTestCase { static int numValueBuckets; static long[] valueCounts; + public PercentilesBucketIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("tag", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java index f5a5d025946ec..b4da63802bc50 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java @@ -32,22 +32,30 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.collect.EvictingQueue; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.max; @@ -61,7 +69,7 @@ import static org.hamcrest.core.IsNull.nullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class SerialDiffIT extends OpenSearchIntegTestCase { +public class SerialDiffIT extends ParameterizedOpenSearchIntegTestCase { private static final String INTERVAL_FIELD = "l_value"; private static final String VALUE_FIELD = "v_value"; @@ -90,6 +98,23 @@ public String toString() { } } + public SerialDiffIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private ValuesSourceAggregationBuilder> randomMetric(String name, String field) { int rand = randomIntBetween(0, 3); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java index e9f34f6aa65d9..21fdd5e761e77 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; @@ -42,11 +46,15 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -58,10 +66,9 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class StatsBucketIT extends OpenSearchIntegTestCase { +public class StatsBucketIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; - static int numDocs; static int interval; static int minRandomValue; @@ -69,6 +76,23 @@ public class StatsBucketIT extends OpenSearchIntegTestCase { static int numValueBuckets; static long[] valueCounts; + public StatsBucketIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("tag", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java index 5bd962017c247..d4bd8f21b2a99 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java @@ -32,8 +32,12 @@ package org.opensearch.search.aggregations.pipeline; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; @@ -42,11 +46,15 @@ import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -58,7 +66,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class SumBucketIT extends OpenSearchIntegTestCase { +public class SumBucketIT extends ParameterizedOpenSearchIntegTestCase { private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; @@ -69,6 +77,23 @@ public class SumBucketIT extends OpenSearchIntegTestCase { static int numValueBuckets; static long[] valueCounts; + public SumBucketIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { assertAcked(client().admin().indices().prepareCreate("idx").setMapping("tag", "type=keyword").get()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java index 6681715981c54..6886f8d67589e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java @@ -32,24 +32,30 @@ package org.opensearch.search.sort; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.Version; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.geo.GeoDistance; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.geometry.utils.Geohash; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFirstHit; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -60,7 +66,24 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -public class GeoDistanceIT extends OpenSearchIntegTestCase { +public class GeoDistanceIT extends ParameterizedOpenSearchIntegTestCase { + + public GeoDistanceIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected boolean forbidPrivateIndexSettings() { diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/bucket/AbstractTermsTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/bucket/AbstractTermsTestCase.java index 75192e276982e..8e94f2cacf070 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/bucket/AbstractTermsTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/bucket/AbstractTermsTestCase.java @@ -32,16 +32,41 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -public abstract class AbstractTermsTestCase extends OpenSearchIntegTestCase { +public abstract class AbstractTermsTestCase extends ParameterizedOpenSearchIntegTestCase { + + public AbstractTermsTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public String randomExecutionHint() { return randomBoolean() ? null : randomFrom(ExecutionMode.values()).toString(); diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/metrics/AbstractNumericTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/metrics/AbstractNumericTestCase.java index a4f6b97115bb0..103b67e2782de 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/metrics/AbstractNumericTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/metrics/AbstractNumericTestCase.java @@ -31,18 +31,43 @@ package org.opensearch.search.aggregations.metrics; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; @OpenSearchIntegTestCase.SuiteScopeTestCase -public abstract class AbstractNumericTestCase extends OpenSearchIntegTestCase { +public abstract class AbstractNumericTestCase extends ParameterizedOpenSearchIntegTestCase { protected static long minValue, maxValue, minValues, maxValues; + public AbstractNumericTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); From d2ca9261892ddc80c32d24870ee89b90e8916e9f Mon Sep 17 00:00:00 2001 From: Neetika Singhal Date: Wed, 13 Sep 2023 15:27:04 -0700 Subject: [PATCH 027/388] Parameterize tests for concurrent search 3 (#9982) Signed-off-by: Neetika Singhal --- .../common/QueryStringWithAnalyzersIT.java | 26 +++++++++++++++-- .../common/HighlighterWithAnalyzersTests.java | 26 +++++++++++++++-- .../geo/GeoModulePluginIntegTestCase.java | 28 +++++++++++++++++-- .../opensearch/geo/search/MissingValueIT.java | 5 ++++ ...AbstractGeoBucketAggregationIntegTest.java | 4 +++ .../aggregations/bucket/GeoHashGridIT.java | 5 ++++ .../aggregations/bucket/GeoTileGridIT.java | 5 ++++ .../aggregations/bucket/ShardReduceIT.java | 5 ++++ ...ractGeoAggregatorModulePluginTestCase.java | 4 +++ .../metrics/GeoBoundsITTestCase.java | 5 ++++ .../metrics/GeoCentroidITTestCase.java | 5 ++++ .../script/expression/MoreExpressionIT.java | 27 ++++++++++++++++-- .../script/expression/StoredExpressionIT.java | 27 ++++++++++++++++-- .../mustache/MultiSearchTemplateIT.java | 26 +++++++++++++++-- .../percolator/PercolatorQuerySearchIT.java | 25 +++++++++++++++-- .../index/rankeval/RankEvalRequestIT.java | 26 +++++++++++++++-- .../node/tasks/CancellableTasksIT.java | 27 ++++++++++++++++-- .../action/termvectors/GetTermVectorsIT.java | 4 +++ .../termvectors/MultiTermVectorsIT.java | 4 +++ .../org/opensearch/index/IndexSortIT.java | 27 ++++++++++++++++-- .../index/search/MatchPhraseQueryIT.java | 28 +++++++++++++++++-- .../index/suggest/stats/SuggestStatsIT.java | 27 +++++++++++++++++- .../org/opensearch/mget/SimpleMgetIT.java | 27 ++++++++++++++++-- .../search/aggregations/EquivalenceIT.java | 27 ++++++++++++++++-- .../bucket/AdjacencyMatrixIT.java | 26 ++++++++++++++++- .../aggregations/bucket/DateHistogramIT.java | 26 ++++++++++++++++- .../bucket/DateHistogramOffsetIT.java | 27 +++++++++++++++++- .../aggregations/bucket/ShardSizeTermsIT.java | 6 ++++ .../bucket/TermsShardMinDocCountIT.java | 28 +++++++++++++++++-- .../search/stats/SearchStatsIT.java | 25 ++++++++++++++++- .../AbstractTermVectorsTestCase.java | 27 ++++++++++++++++-- .../bucket/ShardSizeTestCase.java | 27 ++++++++++++++++-- .../metrics/AbstractGeoTestCase.java | 1 - .../DelayedShardAggregationIT.java | 22 +++++++++++++-- 34 files changed, 595 insertions(+), 40 deletions(-) diff --git a/modules/analysis-common/src/internalClusterTest/java/org/opensearch/analysis/common/QueryStringWithAnalyzersIT.java b/modules/analysis-common/src/internalClusterTest/java/org/opensearch/analysis/common/QueryStringWithAnalyzersIT.java index 785e597857825..71af708f2e1dc 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/opensearch/analysis/common/QueryStringWithAnalyzersIT.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/opensearch/analysis/common/QueryStringWithAnalyzersIT.java @@ -32,20 +32,42 @@ package org.opensearch.analysis.common; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.Operator; import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.Arrays; import java.util.Collection; import static org.opensearch.index.query.QueryBuilders.queryStringQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -public class QueryStringWithAnalyzersIT extends OpenSearchIntegTestCase { +public class QueryStringWithAnalyzersIT extends ParameterizedOpenSearchIntegTestCase { + + public QueryStringWithAnalyzersIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Arrays.asList(CommonAnalysisModulePlugin.class); diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java index 46e57faec3a69..26f4acb2b1e6a 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java @@ -32,8 +32,11 @@ package org.opensearch.analysis.common; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; @@ -41,7 +44,7 @@ import org.opensearch.plugins.Plugin; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.Arrays; @@ -55,6 +58,7 @@ import static org.opensearch.index.query.QueryBuilders.matchPhraseQuery; import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.builder.SearchSourceBuilder.highlight; import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -64,7 +68,25 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; -public class HighlighterWithAnalyzersTests extends OpenSearchIntegTestCase { +public class HighlighterWithAnalyzersTests extends ParameterizedOpenSearchIntegTestCase { + + public HighlighterWithAnalyzersTests(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Arrays.asList(CommonAnalysisModulePlugin.class); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java index b17f4804d4d50..c38b29502e282 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java @@ -8,26 +8,50 @@ package org.opensearch.geo; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.geometry.utils.StandardValidator; import org.opensearch.geometry.utils.WellKnownText; import org.opensearch.index.mapper.GeoShapeFieldMapper; import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.TestGeoShapeFieldMapperPlugin; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; + /** * This is the base class for all the Geo related integration tests. Use this class to add the features and settings * for the test cluster on which integration tests are running. */ -public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCase { +public abstract class GeoModulePluginIntegTestCase extends ParameterizedOpenSearchIntegTestCase { protected static final double GEOHASH_TOLERANCE = 1E-5D; protected static final WellKnownText WKT = new WellKnownText(true, new StandardValidator(true)); + public GeoModulePluginIntegTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + /** * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this * geo plugin is not getting packaged in a zip, we need to load it before the tests run. diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java index a9dd7d1fd22e7..7344903fd5220 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java @@ -10,6 +10,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.settings.Settings; import org.opensearch.geo.GeoModulePluginIntegTestCase; import org.opensearch.geo.search.aggregations.common.GeoBoundsHelper; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; @@ -43,6 +44,10 @@ public class MissingValueIT extends GeoModulePluginIntegTestCase { private GeoPoint bottomRight; private GeoPoint topLeft; + public MissingValueIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected void setupSuiteScopeCluster() throws Exception { assertAcked( diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java index d9ff3e8f473ef..86d8ad2968e7f 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java @@ -67,6 +67,10 @@ public abstract class AbstractGeoBucketAggregationIntegTest extends GeoModulePlu protected final Version version = VersionUtils.randomIndexCompatibleVersion(random()); + public AbstractGeoBucketAggregationIntegTest(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected boolean forbidPrivateIndexSettings() { return false; diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java index 459a0986d3103..4048bb62f8818 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java @@ -35,6 +35,7 @@ import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.common.settings.Settings; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.opensearch.geo.search.aggregations.common.GeoBoundsHelper; @@ -64,6 +65,10 @@ public class GeoHashGridIT extends AbstractGeoBucketAggregationIntegTest { private static final String AGG_NAME = "geohashgrid"; + public GeoHashGridIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override public void setupSuiteScopeCluster() throws Exception { Random random = random(); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridIT.java index 6b09a843af566..2a5772d417530 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridIT.java @@ -12,6 +12,7 @@ import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.common.settings.Settings; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.opensearch.geo.search.aggregations.common.GeoBoundsHelper; @@ -38,6 +39,10 @@ public class GeoTileGridIT extends AbstractGeoBucketAggregationIntegTest { private static final String AGG_NAME = "geotilegrid"; + public GeoTileGridIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override public void setupSuiteScopeCluster() throws Exception { final Random random = random(); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java index d22d2089a3ae3..85541c60f133c 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java @@ -10,6 +10,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.geo.GeoModulePluginIntegTestCase; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.geo.tests.common.AggregationBuilders; @@ -34,6 +35,10 @@ @OpenSearchIntegTestCase.SuiteScopeTestCase public class ShardReduceIT extends GeoModulePluginIntegTestCase { + public ShardReduceIT(Settings dynamicSettings) { + super(dynamicSettings); + } + private IndexRequestBuilder indexDoc(String date, int value) throws Exception { return client().prepareIndex("idx") .setSource( diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java index d76104882d676..711744b944ce3 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java @@ -65,6 +65,10 @@ public abstract class AbstractGeoAggregatorModulePluginTestCase extends GeoModul protected static Map expectedDocCountsForGeoHash = null; protected static Map expectedCentroidsForGeoHash = null; + public AbstractGeoAggregatorModulePluginTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex(UNMAPPED_IDX_NAME); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java index d95cd85b49cd4..1c28df6bc4ea2 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java @@ -34,6 +34,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.util.BigArray; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.global.Global; @@ -61,6 +62,10 @@ public class GeoBoundsITTestCase extends AbstractGeoAggregatorModulePluginTestCase { private static final String aggName = "geoBounds"; + public GeoBoundsITTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false)) diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java index 01d2656adb750..2dc8a91600419 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java @@ -34,6 +34,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.settings.Settings; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.geo.tests.common.AggregationBuilders; import org.opensearch.search.aggregations.metrics.GeoCentroid; @@ -51,6 +52,10 @@ public class GeoCentroidITTestCase extends AbstractGeoAggregatorModulePluginTestCase { private static final String aggName = "geoCentroid"; + public GeoCentroidITTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) .addAggregation( diff --git a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java index 46cac9afa38fc..8ca28a905f216 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java @@ -32,12 +32,16 @@ package org.opensearch.script.expression; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.action.update.UpdateRequestBuilder; import org.opensearch.common.lucene.search.function.CombineFunction; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; @@ -53,9 +57,10 @@ import org.opensearch.search.aggregations.pipeline.SimpleValue; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -64,6 +69,7 @@ import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.bucketScript; @@ -74,7 +80,24 @@ import static org.hamcrest.Matchers.notNullValue; // TODO: please convert to unit tests! -public class MoreExpressionIT extends OpenSearchIntegTestCase { +public class MoreExpressionIT extends ParameterizedOpenSearchIntegTestCase { + + public MoreExpressionIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java index 665ebf3c2caea..b1cb5356a4405 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java @@ -32,7 +32,10 @@ package org.opensearch.script.expression; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.plugins.Plugin; @@ -40,16 +43,36 @@ import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.hamcrest.Matchers.containsString; //TODO: please convert to unit tests! -public class StoredExpressionIT extends OpenSearchIntegTestCase { +public class StoredExpressionIT extends ParameterizedOpenSearchIntegTestCase { + + public StoredExpressionIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/MultiSearchTemplateIT.java index bb11e493ba3d1..e480fbbd22ad2 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/MultiSearchTemplateIT.java @@ -32,12 +32,16 @@ package org.opensearch.script.mustache; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexNotFoundException; import org.opensearch.plugins.Plugin; import org.opensearch.script.ScriptType; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.Arrays; import java.util.Collection; @@ -46,6 +50,7 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; @@ -53,7 +58,24 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.core.Is.is; -public class MultiSearchTemplateIT extends OpenSearchIntegTestCase { +public class MultiSearchTemplateIT extends ParameterizedOpenSearchIntegTestCase { + + public MultiSearchTemplateIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java index b5c082a5667c1..c8763c2f3f749 100644 --- a/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java @@ -31,6 +31,8 @@ package org.opensearch.percolator; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.join.ScoreMode; import org.opensearch.OpenSearchException; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; @@ -39,6 +41,7 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.DistanceUnit; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesArray; @@ -54,7 +57,7 @@ import org.opensearch.plugins.Plugin; import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.Arrays; @@ -77,6 +80,7 @@ import static org.opensearch.index.query.QueryBuilders.spanNotQuery; import static org.opensearch.index.query.QueryBuilders.spanTermQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchHits; @@ -86,7 +90,24 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.IsNull.notNullValue; -public class PercolatorQuerySearchIT extends OpenSearchIntegTestCase { +public class PercolatorQuerySearchIT extends ParameterizedOpenSearchIntegTestCase { + + public PercolatorQuerySearchIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected boolean addMockGeoShapeFieldMapper() { diff --git a/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java index 6eb974c77a5f3..cdc3cac1a1f06 100644 --- a/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java @@ -32,10 +32,14 @@ package org.opensearch.index.rankeval; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -43,7 +47,7 @@ import org.opensearch.indices.IndexClosedException; import org.opensearch.plugins.Plugin; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.Before; import java.util.ArrayList; @@ -54,15 +58,33 @@ import java.util.Set; import static org.opensearch.index.rankeval.EvaluationMetric.filterUnratedDocuments; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.instanceOf; -public class RankEvalRequestIT extends OpenSearchIntegTestCase { +public class RankEvalRequestIT extends ParameterizedOpenSearchIntegTestCase { private static final String TEST_INDEX = "test"; private static final String INDEX_ALIAS = "alias0"; private static final int RELEVANT_RATING_1 = 1; + public RankEvalRequestIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected Collection> nodePlugins() { return Arrays.asList(RankEvalModulePlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index f11c696310b39..c4dcedcc722cf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -31,6 +31,8 @@ package org.opensearch.action.admin.cluster.node.tasks; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionRequest; @@ -49,6 +51,8 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.set.Sets; @@ -65,7 +69,7 @@ import org.opensearch.tasks.TaskInfo; import org.opensearch.tasks.TaskManager; import org.opensearch.test.InternalTestCluster; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportResponseHandler; @@ -74,6 +78,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -86,6 +91,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -93,7 +99,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; -public class CancellableTasksIT extends OpenSearchIntegTestCase { +public class CancellableTasksIT extends ParameterizedOpenSearchIntegTestCase { static int idGenerator = 0; static final Map beforeSendLatches = ConcurrentCollections.newConcurrentMap(); @@ -101,6 +107,23 @@ public class CancellableTasksIT extends OpenSearchIntegTestCase { static final Map beforeExecuteLatches = ConcurrentCollections.newConcurrentMap(); static final Map completedLatches = ConcurrentCollections.newConcurrentMap(); + public CancellableTasksIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Before public void resetTestStates() { idGenerator = 0; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/termvectors/GetTermVectorsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/termvectors/GetTermVectorsIT.java index 9101d0b575ab6..7bd1467933e00 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/termvectors/GetTermVectorsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/termvectors/GetTermVectorsIT.java @@ -74,6 +74,10 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { + public GetTermVectorsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + @Override protected Collection> nodePlugins() { return Collections.singleton(MockKeywordPlugin.class); diff --git a/server/src/internalClusterTest/java/org/opensearch/action/termvectors/MultiTermVectorsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/termvectors/MultiTermVectorsIT.java index 91d280a9c4771..7c6c47c682281 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/termvectors/MultiTermVectorsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/termvectors/MultiTermVectorsIT.java @@ -52,6 +52,10 @@ import static org.hamcrest.Matchers.nullValue; public class MultiTermVectorsIT extends AbstractTermVectorsTestCase { + public MultiTermVectorsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + public void testDuelESLucene() throws Exception { AbstractTermVectorsTestCase.TestFieldSetting[] testFieldSettings = getFieldSettings(); createIndexBasedOnFieldSettings("test", "alias", testFieldSettings); diff --git a/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java b/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java index d547ded8152dd..bb6e356db188f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java @@ -32,22 +32,45 @@ package org.opensearch.index; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.hamcrest.Matchers.containsString; -public class IndexSortIT extends OpenSearchIntegTestCase { +public class IndexSortIT extends ParameterizedOpenSearchIntegTestCase { private static final XContentBuilder TEST_MAPPING = createTestMapping(); + public IndexSortIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private static XContentBuilder createTestMapping() { try { return jsonBuilder().startObject() diff --git a/server/src/internalClusterTest/java/org/opensearch/index/search/MatchPhraseQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/index/search/MatchPhraseQueryIT.java index 6d76ee48a5b95..2d28578dbebcc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/search/MatchPhraseQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/search/MatchPhraseQueryIT.java @@ -32,26 +32,50 @@ package org.opensearch.index.search; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.MatchPhraseQueryBuilder; import org.opensearch.index.search.MatchQuery.ZeroTermsQuery; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.concurrent.ExecutionException; import static org.opensearch.index.query.QueryBuilders.matchPhraseQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -public class MatchPhraseQueryIT extends OpenSearchIntegTestCase { +public class MatchPhraseQueryIT extends ParameterizedOpenSearchIntegTestCase { + private static final String INDEX = "test"; + public MatchPhraseQueryIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Before public void setUp() throws Exception { super.setUp(); diff --git a/server/src/internalClusterTest/java/org/opensearch/index/suggest/stats/SuggestStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/index/suggest/stats/SuggestStatsIT.java index 9940b1eb13a52..6332b1b97426f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/suggest/stats/SuggestStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/suggest/stats/SuggestStatsIT.java @@ -32,6 +32,8 @@ package org.opensearch.index.suggest.stats; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; @@ -42,17 +44,22 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.search.stats.SearchStats; import org.opensearch.search.suggest.SuggestBuilder; import org.opensearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.opensearch.search.suggest.term.TermSuggestionBuilder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.Set; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.hamcrest.Matchers.equalTo; @@ -61,7 +68,25 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; @OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) -public class SuggestStatsIT extends OpenSearchIntegTestCase { +public class SuggestStatsIT extends ParameterizedOpenSearchIntegTestCase { + + public SuggestStatsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected int numberOfReplicas() { return 0; diff --git a/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java b/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java index 9e13f06983860..f77ae80a55276 100644 --- a/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/mget/SimpleMgetIT.java @@ -32,6 +32,8 @@ package org.opensearch.mget; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.get.MultiGetItemResponse; @@ -40,16 +42,20 @@ import org.opensearch.action.get.MultiGetResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.search.fetch.subphase.FetchSourceContext; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import java.util.Map; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -57,7 +63,24 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -public class SimpleMgetIT extends OpenSearchIntegTestCase { +public class SimpleMgetIT extends ParameterizedOpenSearchIntegTestCase { + + public SimpleMgetIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } public void testThatMgetShouldWorkWithOneIndexMissing() throws IOException { createIndex("test"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java index 21f833d5430db..2ffdf5fb32778 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java @@ -32,10 +32,14 @@ package org.opensearch.search.aggregations; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; @@ -52,11 +56,12 @@ import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregatorFactory; import org.opensearch.search.aggregations.metrics.Sum; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.After; import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -68,6 +73,7 @@ import java.util.function.Function; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.extendedStats; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -88,7 +94,24 @@ * Additional tests that aim at testing more complex aggregation trees on larger random datasets, so that things like * the growth of dynamic arrays is tested. */ -public class EquivalenceIT extends OpenSearchIntegTestCase { +public class EquivalenceIT extends ParameterizedOpenSearchIntegTestCase { + + public EquivalenceIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/AdjacencyMatrixIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/AdjacencyMatrixIT.java index 011ebf8add92a..cd0922606ec99 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/AdjacencyMatrixIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/AdjacencyMatrixIT.java @@ -32,11 +32,14 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.query.BoolQueryBuilder; @@ -47,15 +50,19 @@ import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.metrics.Avg; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.adjacencyMatrix; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -68,11 +75,28 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class AdjacencyMatrixIT extends OpenSearchIntegTestCase { +public class AdjacencyMatrixIT extends ParameterizedOpenSearchIntegTestCase { static int numDocs, numSingleTag1Docs, numSingleTag2Docs, numTag1Docs, numTag2Docs, numMultiTagDocs; static final int MAX_NUM_FILTERS = 3; + public AdjacencyMatrixIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java index ec7278f74e8af..4ce8af3e0f081 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java @@ -31,6 +31,8 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; @@ -39,6 +41,7 @@ import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.DateMathParser; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -56,6 +59,7 @@ import org.opensearch.search.aggregations.metrics.Avg; import org.opensearch.search.aggregations.metrics.Sum; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import org.junit.After; @@ -78,6 +82,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -93,7 +98,7 @@ import static org.hamcrest.core.IsNull.notNullValue; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class DateHistogramIT extends OpenSearchIntegTestCase { +public class DateHistogramIT extends ParameterizedOpenSearchIntegTestCase { static Map> expectedMultiSortBuckets; @@ -101,6 +106,23 @@ private ZonedDateTime date(int month, int day) { return ZonedDateTime.of(2012, month, day, 0, 0, 0, 0, ZoneOffset.UTC); } + public DateHistogramIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private ZonedDateTime date(String date) { return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); } @@ -1733,6 +1755,7 @@ public void testScriptCaching() throws Exception { .getMissCount(), equalTo(2L) ); + cluster().wipeIndices("cache_test_idx"); } public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAndKeyDesc() throws Exception { @@ -1850,6 +1873,7 @@ public void testDateNanosHistogram() throws Exception { assertEquals(1, buckets.get(0).getDocCount()); assertEquals(946771200000L, ((ZonedDateTime) buckets.get(1).getKey()).toEpochSecond() * 1000); assertEquals(1, buckets.get(1).getDocCount()); + cluster().wipeIndices("nanos"); } public void testDateKeyFormatting() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 19e5bdb8916b8..04115f69172da 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -31,24 +31,32 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.junit.After; import org.junit.Before; import java.io.IOException; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.IsNull.notNullValue; @@ -61,11 +69,28 @@ */ @OpenSearchIntegTestCase.SuiteScopeTestCase @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class DateHistogramOffsetIT extends OpenSearchIntegTestCase { +public class DateHistogramOffsetIT extends ParameterizedOpenSearchIntegTestCase { private static final String DATE_FORMAT = "yyyy-MM-dd:hh-mm-ss"; private static final DateFormatter FORMATTER = DateFormatter.forPattern(DATE_FORMAT); + public DateHistogramOffsetIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private ZonedDateTime date(String date) { return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardSizeTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardSizeTermsIT.java index c89a694271703..145830f02ee56 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardSizeTermsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardSizeTermsIT.java @@ -32,6 +32,7 @@ package org.opensearch.search.aggregations.bucket; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.terms.Terms; @@ -45,6 +46,11 @@ import static org.hamcrest.Matchers.equalTo; public class ShardSizeTermsIT extends ShardSizeTestCase { + + public ShardSizeTermsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + public void testNoShardSizeString() throws Exception { createIdx("type=keyword"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java index e7e826d981c84..b0d8e7ea02e8f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsShardMinDocCountIT.java @@ -31,9 +31,12 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.BucketOrder; @@ -41,13 +44,16 @@ import org.opensearch.search.aggregations.bucket.terms.SignificantTerms; import org.opensearch.search.aggregations.bucket.terms.SignificantTermsAggregatorFactory; import org.opensearch.search.aggregations.bucket.terms.Terms; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.significantTerms; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -55,9 +61,27 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; -public class TermsShardMinDocCountIT extends OpenSearchIntegTestCase { +public class TermsShardMinDocCountIT extends ParameterizedOpenSearchIntegTestCase { + private static final String index = "someindex"; + public TermsShardMinDocCountIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private static String randomExecutionHint() { return randomBoolean() ? null : randomFrom(SignificantTermsAggregatorFactory.ExecutionMode.values()).toString(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java index c72b5d40553b3..23d48b173a3db 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.stats; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; @@ -42,6 +44,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.search.stats.SearchStats.Stats; import org.opensearch.plugins.Plugin; @@ -50,7 +53,9 @@ import org.opensearch.script.ScriptType; import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -61,6 +66,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -73,7 +79,24 @@ import static org.hamcrest.Matchers.nullValue; @OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) -public class SearchStatsIT extends OpenSearchIntegTestCase { +public class SearchStatsIT extends ParameterizedOpenSearchIntegTestCase { + + public SearchStatsIT(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java b/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java index 23153b5a45d4c..4b0bde0984ad1 100644 --- a/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java +++ b/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java @@ -32,6 +32,8 @@ package org.opensearch.action.termvectors; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.LowerCaseFilter; @@ -62,22 +64,43 @@ import org.apache.lucene.store.Directory; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -public abstract class AbstractTermVectorsTestCase extends OpenSearchIntegTestCase { +public abstract class AbstractTermVectorsTestCase extends ParameterizedOpenSearchIntegTestCase { + + public AbstractTermVectorsTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + protected static class TestFieldSetting { public final String name; public final boolean storedOffset; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/ShardSizeTestCase.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/ShardSizeTestCase.java index d6981d1c34652..21d05305eed1b 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/ShardSizeTestCase.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/ShardSizeTestCase.java @@ -32,21 +32,44 @@ package org.opensearch.search.aggregations.bucket; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.is; -public abstract class ShardSizeTestCase extends OpenSearchIntegTestCase { +public abstract class ShardSizeTestCase extends ParameterizedOpenSearchIntegTestCase { + + public ShardSizeTestCase(Settings dynamicSettings) { + super(dynamicSettings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected int numberOfShards() { diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java index e0c8c93c1037a..e02c00005df9b 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -76,7 +76,6 @@ public abstract class AbstractGeoTestCase extends ParameterizedOpenSearchIntegTe protected static final String DATELINE_IDX_NAME = "dateline_idx"; protected static final String HIGH_CARD_IDX_NAME = "high_card_idx"; protected static final String IDX_ZERO_NAME = "idx_zero"; - protected static int numDocs; protected static int numUniqueGeoPoints; protected static GeoPoint[] singleValues, multiValues; diff --git a/test/external-modules/delayed-aggs/src/internalClusterTest/java/org/opensearch/search/aggregations/DelayedShardAggregationIT.java b/test/external-modules/delayed-aggs/src/internalClusterTest/java/org/opensearch/search/aggregations/DelayedShardAggregationIT.java index e850ae9dcc859..90f4f1ba2ceb2 100644 --- a/test/external-modules/delayed-aggs/src/internalClusterTest/java/org/opensearch/search/aggregations/DelayedShardAggregationIT.java +++ b/test/external-modules/delayed-aggs/src/internalClusterTest/java/org/opensearch/search/aggregations/DelayedShardAggregationIT.java @@ -31,25 +31,43 @@ package org.opensearch.search.aggregations; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.plugins.Plugin; import org.opensearch.search.aggregations.bucket.filter.InternalFilter; import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class DelayedShardAggregationIT extends OpenSearchIntegTestCase { +public class DelayedShardAggregationIT extends ParameterizedOpenSearchIntegTestCase { + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { From 921cd0c44640e1a16a715fb79072a5e53950086f Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Thu, 14 Sep 2023 08:46:05 +0800 Subject: [PATCH 028/388] Fix ignore_missing parameter has no effect in rename ingest processor (#9725) * Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor Signed-off-by: Gao Binlong * Small change Co-authored-by: Andriy Redko Signed-off-by: gaobinlong --------- Signed-off-by: Gao Binlong Signed-off-by: gaobinlong Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + .../ingest/common/RenameProcessor.java | 6 +- .../ingest/common/RenameProcessorTests.java | 14 ++++ .../test/ingest/280_rename_processor.yml | 66 +++++++++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index ee0b28aaeaf00..0a52c3d7f2a3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Removed ### Fixed +- Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725)) ### Security diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java index af356eb10d79c..7564bbdf95f45 100644 --- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java +++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java @@ -32,6 +32,7 @@ package org.opensearch.ingest.common; +import org.opensearch.core.common.Strings; import org.opensearch.ingest.AbstractProcessor; import org.opensearch.ingest.ConfigurationUtils; import org.opensearch.ingest.IngestDocument; @@ -80,9 +81,12 @@ boolean isIgnoreMissing() { @Override public IngestDocument execute(IngestDocument document) { String path = document.renderTemplate(field); - if (document.hasField(path, true) == false) { + final boolean fieldPathIsNullOrEmpty = Strings.isNullOrEmpty(path); + if (fieldPathIsNullOrEmpty || document.hasField(path, true) == false) { if (ignoreMissing) { return document; + } else if (fieldPathIsNullOrEmpty) { + throw new IllegalArgumentException("field path cannot be null nor empty"); } else { throw new IllegalArgumentException("field [" + path + "] doesn't exist"); } diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java index fc95693024cb0..a600464371af8 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java @@ -112,6 +112,15 @@ public void testRenameNonExistingField() throws Exception { } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("field [" + fieldName + "] doesn't exist")); } + + // when using template snippet, the resolved field path maybe empty + processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), false); + try { + processor.execute(ingestDocument); + fail("processor execute should have failed"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo("field path cannot be null nor empty")); + } } public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception { @@ -121,6 +130,11 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception { Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), true); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); + + // when using template snippet, the resolved field path maybe empty + processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), true); + processor.execute(ingestDocument); + assertIngestDocument(originalIngestDocument, ingestDocument); } public void testRenameNewFieldAlreadyExists() throws Exception { diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml new file mode 100644 index 0000000000000..96b2256bcc1dc --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml @@ -0,0 +1,66 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + +--- +"Test rename processor with non-existing field and without ignore_missing": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "rename" : { + "field" : "{{field_foo}}", + "target_field" : "bar" + } + } + ] + } + - match: { acknowledged: true } + + - do: + catch: '/field path cannot be null nor empty/' + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: { message: "foo bar baz" } + +--- +"Test rename processor with non-existing field and ignore_missing": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "rename" : { + "field" : "{{field_foo}}", + "target_field" : "bar", + "ignore_missing" : true + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: { message: "foo bar baz" } + + - do: + get: + index: test + id: 1 + - match: { _source.message: "foo bar baz" } From 28724ce97704272fb55a013da59d5ec3b301978a Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Wed, 13 Sep 2023 22:31:15 -0700 Subject: [PATCH 029/388] Fix typo in Precommit and Assemble workflow (#10037) --- .github/workflows/precommit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index f562b46d5b124..c628f48e57eba 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -1,4 +1,4 @@ -name: Gradle Precommit and Asssemble +name: Gradle Precommit and Assemble on: [pull_request] jobs: From 71f6136db5f6fe8661a6dd5ec46ed7c6336ba63b Mon Sep 17 00:00:00 2001 From: Laurent Laborde Date: Thu, 14 Sep 2023 18:47:50 +0200 Subject: [PATCH 030/388] Fixing javadoc error (#9976) * Fixing javadoc error fixing javadoc error and warning javadoc for org.opensearch.core.indices.breaker javadoc for org.opensearch.core.index.shard.ShardId javadoc for org.opensearch.core.index.Index fixing a mishap in formatting rule fixing javadoc of org.opensearch.cli.Terminal Signed-off-by: Laurent Laborde * removing unsupported @ImplNote Signed-off-by: Laurent Laborde * Update libs/common/src/main/java/org/opensearch/common/collect/Iterators.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update libs/compress/src/main/java/org/opensearch/compress/ZstdCompressor.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update CollectionUtils.java Signed-off-by: Laurent Laborde * fixing javadoc warnings Signed-off-by: Laurent Laborde * remove useless p tag Signed-off-by: Laurent Laborde * removing a p tag Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * proper capitalization Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde --------- Signed-off-by: Laurent Laborde Co-authored-by: Andriy Redko --- .../opensearch/common/collect/Iterators.java | 19 ++++++ .../opensearch/compress/ZstdCompressor.java | 51 ++++++++++++-- .../core/common/breaker/CircuitBreaker.java | 28 +++++--- .../breaker/CircuitBreakingException.java | 4 ++ .../common/breaker/NoopCircuitBreaker.java | 65 ++++++++++++++++-- .../common/bytes/AbstractBytesReference.java | 3 +- .../common/logging/LoggerMessageFormat.java | 29 ++++++++ .../common/transport/TransportAddress.java | 12 +++- .../core/common/unit/ByteSizeUnit.java | 7 ++ .../core/common/util/CollectionUtils.java | 49 ++++++++++++-- .../java/org/opensearch/core/index/Index.java | 52 +++++++++++++-- .../opensearch/core/index/shard/ShardId.java | 60 +++++++++++++++++ .../breaker/AllCircuitBreakerStats.java | 26 ++++++++ .../indices/breaker/CircuitBreakerStats.java | 56 +++++++++++++++- .../breaker/NoneCircuitBreakerService.java | 15 ++++- .../java/org/opensearch/geometry/Circle.java | 66 +++++++++++++++++++ 16 files changed, 512 insertions(+), 30 deletions(-) diff --git a/libs/common/src/main/java/org/opensearch/common/collect/Iterators.java b/libs/common/src/main/java/org/opensearch/common/collect/Iterators.java index c7e7ae6a44a21..9b64932356c10 100644 --- a/libs/common/src/main/java/org/opensearch/common/collect/Iterators.java +++ b/libs/common/src/main/java/org/opensearch/common/collect/Iterators.java @@ -41,6 +41,15 @@ * @opensearch.internal */ public class Iterators { + + /** + * Concat iterators + * + * @param iterators the iterators to concat + * @param the type of iterator + * @return a new {@link ConcatenatedIterator} + * @throws NullPointerException if iterators is null + */ public static Iterator concat(Iterator... iterators) { if (iterators == null) { throw new NullPointerException("iterators"); @@ -71,6 +80,11 @@ static class ConcatenatedIterator implements Iterator { this.iterators = iterators; } + /** + * Returns {@code true} if the iteration has more elements. (In other words, returns {@code true} if {@link #next} would return an + * element rather than throwing an exception.) + * @return {@code true} if the iteration has more elements + */ @Override public boolean hasNext() { boolean hasNext = false; @@ -81,6 +95,11 @@ public boolean hasNext() { return hasNext; } + /** + * Returns the next element in the iteration. + * @return the next element in the iteration + * @throws NoSuchElementException if the iteration has no more elements + */ @Override public T next() { if (!hasNext()) { diff --git a/libs/compress/src/main/java/org/opensearch/compress/ZstdCompressor.java b/libs/compress/src/main/java/org/opensearch/compress/ZstdCompressor.java index 01afc368fb120..e2a740f72be93 100644 --- a/libs/compress/src/main/java/org/opensearch/compress/ZstdCompressor.java +++ b/libs/compress/src/main/java/org/opensearch/compress/ZstdCompressor.java @@ -30,10 +30,13 @@ * @opensearch.experimental - class methods might change */ public class ZstdCompressor implements Compressor { - // An arbitrary header that we use to identify compressed streams - // It needs to be different from other compressors and to not be specific - // enough so that no stream starting with these bytes could be detected as - // a XContent + + /** + * An arbitrary header that we use to identify compressed streams + * It needs to be different from other compressors and to not be specific + * enough so that no stream starting with these bytes could be detected as + * a XContent + * */ private static final byte[] HEADER = new byte[] { 'Z', 'S', 'T', 'D', '\0' }; /** @@ -44,10 +47,20 @@ public class ZstdCompressor implements Compressor { @PublicApi(since = "2.10.0") public static final String NAME = "ZSTD"; + /** + * The compression level for {@link ZstdOutputStreamNoFinalizer} + */ private static final int LEVEL = 3; + /** The buffer size for {@link BufferedInputStream} and {@link BufferedOutputStream} + */ private static final int BUFFER_SIZE = 4096; + /** + * Compares the given bytes with the {@link ZstdCompressor#HEADER} of a compressed stream + * @param bytes the bytes to compare to ({@link ZstdCompressor#HEADER}) + * @return true if the bytes are the {@link ZstdCompressor#HEADER}, false otherwise + */ @Override public boolean isCompressed(BytesReference bytes) { if (bytes.length() < HEADER.length) { @@ -61,11 +74,22 @@ public boolean isCompressed(BytesReference bytes) { return true; } + /** + * Returns the length of the {@link ZstdCompressor#HEADER} + * @return the {@link ZstdCompressor#HEADER} length + */ @Override public int headerLength() { return HEADER.length; } + /** + * Returns a new {@link ZstdInputStreamNoFinalizer} from the given compressed {@link InputStream} + * @param in the compressed {@link InputStream} + * @return a new {@link ZstdInputStreamNoFinalizer} from the given compressed {@link InputStream} + * @throws IOException if an I/O error occurs + * @throws IllegalArgumentException if the input stream is not compressed with ZSTD + */ @Override public InputStream threadLocalInputStream(InputStream in) throws IOException { final byte[] header = in.readNBytes(HEADER.length); @@ -75,17 +99,36 @@ public InputStream threadLocalInputStream(InputStream in) throws IOException { return new ZstdInputStreamNoFinalizer(new BufferedInputStream(in, BUFFER_SIZE), RecyclingBufferPool.INSTANCE); } + /** + * Returns a new {@link ZstdOutputStreamNoFinalizer} from the given {@link OutputStream} + * @param out the {@link OutputStream} + * @return a new {@link ZstdOutputStreamNoFinalizer} from the given {@link OutputStream} + * @throws IOException if an I/O error occurs + */ @Override public OutputStream threadLocalOutputStream(OutputStream out) throws IOException { out.write(HEADER); return new ZstdOutputStreamNoFinalizer(new BufferedOutputStream(out, BUFFER_SIZE), RecyclingBufferPool.INSTANCE, LEVEL); } + /** + * Always throws an {@link UnsupportedOperationException} as ZSTD compression is supported only for snapshotting + * @param bytesReference a reference to the bytes to uncompress + * @return always throws an exception + * @throws UnsupportedOperationException if the method is called + * @throws IOException is never thrown + */ @Override public BytesReference uncompress(BytesReference bytesReference) throws IOException { throw new UnsupportedOperationException("ZSTD compression is supported only for snapshotting"); } + /** + * Always throws an {@link UnsupportedOperationException} as ZSTD compression is supported only for snapshotting + * @param bytesReference a reference to the bytes to compress + * @return always throws an exception + * @throws UnsupportedOperationException if the method is called + */ @Override public BytesReference compress(BytesReference bytesReference) throws IOException { throw new UnsupportedOperationException("ZSTD compression is supported only for snapshotting"); diff --git a/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java index 0f75f763d21c1..846950ff17c63 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java +++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java @@ -71,17 +71,23 @@ public interface CircuitBreaker { /** * The type of breaker - * + * can be {@link #MEMORY}, {@link #PARENT}, or {@link #NOOP} * @opensearch.internal */ enum Type { - // A regular or ChildMemoryCircuitBreaker + /** A regular or ChildMemoryCircuitBreaker */ MEMORY, - // A special parent-type for the hierarchy breaker service + /** A special parent-type for the hierarchy breaker service */ PARENT, - // A breaker where every action is a noop, it never breaks + /** A breaker where every action is a noop, it never breaks */ NOOP; + /** + * Converts string (case-insensitive) to breaker {@link Type} + * @param value "noop", "parent", or "memory" (case-insensitive) + * @return the breaker {@link Type} + * @throws IllegalArgumentException if value is not "noop", "parent", or "memory" + */ public static Type parseValue(String value) { switch (value.toLowerCase(Locale.ROOT)) { case "noop": @@ -98,13 +104,13 @@ public static Type parseValue(String value) { /** * The breaker durability - * + * can be {@link #TRANSIENT} or {@link #PERMANENT} * @opensearch.internal */ enum Durability { - // The condition that tripped the circuit breaker fixes itself eventually. + /** The condition that tripped the circuit breaker fixes itself eventually. */ TRANSIENT, - // The condition that tripped the circuit breaker requires manual intervention. + /** The condition that tripped the circuit breaker requires manual intervention. */ PERMANENT } @@ -120,11 +126,14 @@ enum Durability { * @param bytes number of bytes to add * @param label string label describing the bytes being added * @return the number of "used" bytes for the circuit breaker + * @throws CircuitBreakingException if the breaker tripped */ double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException; /** * Adjust the circuit breaker without tripping + * @param bytes number of bytes to add + * @return the number of "used" bytes for the circuit breaker */ long addWithoutBreaking(long bytes); @@ -154,7 +163,10 @@ enum Durability { String getName(); /** - * @return whether a tripped circuit breaker will reset itself (transient) or requires manual intervention (permanent). + * Returns the {@link Durability} of this breaker + * @return whether a tripped circuit breaker will + * reset itself ({@link Durability#TRANSIENT}) + * or requires manual intervention ({@link Durability#PERMANENT}). */ Durability getDurability(); diff --git a/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java index e6443a0d48ce0..2df116dcad076 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java +++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java @@ -46,8 +46,11 @@ */ public class CircuitBreakingException extends OpenSearchException { + /** The number of bytes wanted */ private final long bytesWanted; + /** The circuit breaker limit */ private final long byteLimit; + /** The {@link CircuitBreaker.Durability} of the circuit breaker */ private final CircuitBreaker.Durability durability; public CircuitBreakingException(StreamInput in) throws IOException { @@ -88,6 +91,7 @@ public CircuitBreaker.Durability getDurability() { return durability; } + /** Always returns {@link RestStatus#TOO_MANY_REQUESTS} */ @Override public RestStatus status() { return RestStatus.TOO_MANY_REQUESTS; diff --git a/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java index 86a0a7ccb96fd..17b9fefd27c99 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java +++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java @@ -33,65 +33,120 @@ package org.opensearch.core.common.breaker; /** - * A CircuitBreaker that doesn't increment or adjust, and all operations are - * basically noops - * + * A {@link CircuitBreaker} that doesn't increment or adjust, and all operations are + * basically noops. + * It never trips, limit is always -1, always returns 0 for all metrics. * @opensearch.internal */ public class NoopCircuitBreaker implements CircuitBreaker { - public static final int LIMIT = -1; + /** The limit of this breaker is always -1 */ + public static final int LIMIT = -1; + /** Name of this breaker */ private final String name; + /** + * Creates a new NoopCircuitBreaker (that never trip) with the given name + * @param name the name of this breaker + */ public NoopCircuitBreaker(String name) { this.name = name; } + /** + * This is a noop, a noop breaker never trip + * @param fieldName name of this noop breaker + * @param bytesNeeded bytes needed + */ @Override public void circuitBreak(String fieldName, long bytesNeeded) { // noop } + /** + * This is a noop, always return 0 and never throw/trip + * @param bytes number of bytes to add + * @param label string label describing the bytes being added + * @return always return 0 + * @throws CircuitBreakingException never thrown + */ @Override public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException { return 0; } + /** + * This is a noop, nothing is added, always return 0 + * @param bytes number of bytes to add (ignored) + * @return always return 0 + */ @Override public long addWithoutBreaking(long bytes) { return 0; } + /** + * This is a noop, always return 0 + * @return always return 0 + */ @Override public long getUsed() { return 0; } + /** + * A noop breaker have a constant limit of -1 + * @return always return -1 + */ @Override public long getLimit() { return LIMIT; } + /** + * A noop breaker have no overhead, always return 0 + * @return always return 0 + */ @Override public double getOverhead() { return 0; } + /** + * A noop breaker never trip, always return 0 + * @return always return 0 + */ @Override public long getTrippedCount() { return 0; } + /** + * return the name of this breaker + * @return the name of this breaker + */ @Override public String getName() { return this.name; } + /** + * A noop breaker {@link Durability} is always {@link Durability#PERMANENT} + * @return always return {@link Durability#PERMANENT } + */ @Override public Durability getDurability() { return Durability.PERMANENT; } + /** + * Limit and overhead are constant for a noop breaker. + * this is a noop. + * @param limit the desired limit (ignored) + * @param overhead the desired overhead (ignored) + */ @Override - public void setLimitAndOverhead(long limit, double overhead) {} + public void setLimitAndOverhead(long limit, double overhead) { + // noop + } } diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java index e054776d67fdc..8c1efcd00c24e 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java +++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java @@ -49,7 +49,8 @@ */ public abstract class AbstractBytesReference implements BytesReference { - private Integer hash = null; // we cache the hash of this reference since it can be quite costly to re-calculated it + /** we cache the hash of this reference since it can be quite costly to re-calculated it */ + private Integer hash = null; private static final int MAX_UTF16_LENGTH = Integer.MAX_VALUE >> 1; @Override diff --git a/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java b/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java index 59492193d16dc..cd75bddd680e5 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java +++ b/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java @@ -37,6 +37,10 @@ /** * Format string for OpenSearch log messages. + *

+ * This class is almost a copy of {@code org.slf4j.helpers.MessageFormatter}

+ * The original code is licensed under the MIT License and is available at : + * MessageFormatter.java * * @opensearch.internal */ @@ -51,6 +55,17 @@ public static String format(final String messagePattern, final Object... argArra return format(null, messagePattern, argArray); } + /** + * (this is almost a copy of {@code org.slf4j.helpers.MessageFormatter.arrayFormat}) + * + * @param prefix the prefix to prepend to the formatted message (can be null) + * @param messagePattern the message pattern which will be parsed and formatted + * @param argArray an array of arguments to be substituted in place of formatting anchors + * @return null if messagePattern is null

+ * messagePattern if argArray is (null or empty) and prefix is null

+ * prefix + messagePattern if argArray is (null or empty) and prefix is not null

+ * formatted message otherwise (even if prefix is null) + */ public static String format(final String prefix, final String messagePattern, final Object... argArray) { if (messagePattern == null) { return null; @@ -110,6 +125,13 @@ public static String format(final String prefix, final String messagePattern, fi return sbuf.toString(); } + /** + * Checks if (delimterStartIndex - 1) in messagePattern is an escape character. + * @param messagePattern the message pattern + * @param delimiterStartIndex the index of the character to check + * @return true if there is an escape char before the character at delimiterStartIndex.

+ * Always returns false if delimiterStartIndex == 0 (edge case) + */ static boolean isEscapedDelimiter(String messagePattern, int delimiterStartIndex) { if (delimiterStartIndex == 0) { @@ -119,6 +141,13 @@ static boolean isEscapedDelimiter(String messagePattern, int delimiterStartIndex return potentialEscape == ESCAPE_CHAR; } + /** + * Checks if (delimterStartIndex - 2) in messagePattern is an escape character. + * @param messagePattern the message pattern + * @param delimiterStartIndex the index of the character to check + * @return true if (delimterStartIndex - 2) in messagePattern is an escape character. + * Always returns false if delimiterStartIndex is less than 2 (edge case) + */ static boolean isDoubleEscaped(String messagePattern, int delimiterStartIndex) { return delimiterStartIndex >= 2 && messagePattern.charAt(delimiterStartIndex - 2) == ESCAPE_CHAR; } diff --git a/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java b/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java index 1a853877ed0b9..551504ed6f719 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java +++ b/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java @@ -71,6 +71,12 @@ public TransportAddress(InetAddress address, int port) { this(new InetSocketAddress(address, port)); } + /** + * Creates a new {@link TransportAddress} from a {@link InetSocketAddress}. + * @param address the address to wrap + * @throws IllegalArgumentException if the address is null or not resolved + * @see InetSocketAddress#getAddress() + */ public TransportAddress(InetSocketAddress address) { if (address == null) { throw new IllegalArgumentException("InetSocketAddress must not be null"); @@ -82,7 +88,9 @@ public TransportAddress(InetSocketAddress address) { } /** - * Read from a stream. + * Creates a new {@link TransportAddress} from a {@link StreamInput}. + * @param in the stream to read from + * @throws IOException if an I/O error occurs */ public TransportAddress(StreamInput in) throws IOException { final int len = in.readByte(); @@ -116,6 +124,8 @@ public String getAddress() { /** * Returns the addresses port + * @return the port number, or 0 if the socket is not bound yet. + * @see InetSocketAddress#getPort() */ public int getPort() { return address.getPort(); diff --git a/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java index 1f49a3531986c..c15db75d06d49 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java +++ b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java @@ -46,6 +46,13 @@ * helps organize and use size representations that may be maintained * separately across various contexts. * + * It use conventional data storage values (base-2) : + *

    + *
  • 1KB = 1024 bytes
    • + *
  • 1MB = 1024KB
    • + *
  • ...
    • + *
+ * * @opensearch.api */ @PublicApi(since = "1.0.0") diff --git a/libs/core/src/main/java/org/opensearch/core/common/util/CollectionUtils.java b/libs/core/src/main/java/org/opensearch/core/common/util/CollectionUtils.java index e8dd31fcf1869..5335c98182b64 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/util/CollectionUtils.java +++ b/libs/core/src/main/java/org/opensearch/core/common/util/CollectionUtils.java @@ -73,6 +73,16 @@ public static boolean isEmpty(Object[] array) { /** * Return a rotated view of the given list with the given distance. + *
    + *
  • The distance can be negative, in which case the list is rotated to the left.
    • + *
  • The distance can be larger than the size of the list, in which case the list is rotated multiple times.
    • + *
  • The distance can be zero, in which case the list is not rotated.
    • + *
  • The list can be empty, in which case it remains empty.
    • + *
+ * @param list the list to rotate + * @param distance the distance to rotate (positive rotates right, negative rotates left) + * @return a rotated view of the given list with the given distance + * @see RotatedList */ public static List rotate(final List list, int distance) { if (list.isEmpty()) { @@ -92,7 +102,13 @@ public static List rotate(final List list, int distance) { } /** - * in place de-duplicates items in a list + * In place de-duplicates items in a list + * Noop if the list is empty or has one item. + * + * @throws NullPointerException if the list is `null` or comparator is `null` + * @param array the list to de-duplicate + * @param comparator the comparator to use to compare items + * @param the type of the items in the list */ public static void sortAndDedup(final List array, Comparator comparator) { // base case: one item @@ -115,6 +131,12 @@ public static void sortAndDedup(final List array, Comparator comparato array.subList(deduped.nextIndex(), array.size()).clear(); } + /** + * Converts a collection of Integers to an array of ints. + * @param ints The collection of Integers to convert + * @return The array of ints + * @throws NullPointerException if ints is null + */ public static int[] toArray(Collection ints) { Objects.requireNonNull(ints); return ints.stream().mapToInt(s -> s).toArray(); @@ -134,6 +156,11 @@ public static void ensureNoSelfReferences(Object value, String messageHint) { } } + /** + * Converts an object to an Iterable, if possible. + * @param value The object to convert + * @return The Iterable, or null if the object cannot be converted + */ @SuppressWarnings("unchecked") private static Iterable convert(Object value) { if (value == null) { @@ -192,6 +219,13 @@ private static class RotatedList extends AbstractList implements RandomAcc private final List in; private final int distance; + /** + * Creates a rotated list + * @param list The list to rotate + * @param distance The distance to rotate to the right + * @throws IllegalArgumentException if the distance is negative or greater than the size of the list; + * or if the list is not a {@link RandomAccess} list + */ RotatedList(List list, int distance) { if (distance < 0 || distance >= list.size()) { throw new IllegalArgumentException(); @@ -218,6 +252,13 @@ public int size() { } } + /** + * Converts an {@link Iterable} to an {@link ArrayList}. + * @param elements The iterable to convert + * @param the type the elements + * @return an {@link ArrayList} + * @throws NullPointerException if elements is null + */ @SuppressWarnings("unchecked") public static ArrayList iterableAsArrayList(Iterable elements) { if (elements == null) { @@ -297,11 +338,11 @@ public static List> eagerPartition(List list, int size) { } /** - * Check if a collection is empty or not. Empty collection mean either it is null or it has no elements in it. If - * collection contains a null element it means it is not empty. + * Checks if a collection is empty or not. Empty collection mean either it is null or it has no elements in it. + * If collection contains a null element it means it is not empty. * * @param collection {@link Collection} - * @return boolean + * @return true if collection is null or {@code isEmpty()}, false otherwise * @param Element */ public static boolean isEmpty(final Collection collection) { diff --git a/libs/core/src/main/java/org/opensearch/core/index/Index.java b/libs/core/src/main/java/org/opensearch/core/index/Index.java index fdff43f3c9139..77cc628213df9 100644 --- a/libs/core/src/main/java/org/opensearch/core/index/Index.java +++ b/libs/core/src/main/java/org/opensearch/core/index/Index.java @@ -49,6 +49,8 @@ /** * A value class representing the basic required properties of an OpenSearch index. * + * (This class is immutable.) + * * @opensearch.api */ @PublicApi(since = "1.0.0") @@ -57,6 +59,7 @@ public class Index implements Writeable, ToXContentObject { public static final Index[] EMPTY_ARRAY = new Index[0]; private static final String INDEX_UUID_KEY = "index_uuid"; private static final String INDEX_NAME_KEY = "index_name"; + private static final ObjectParser INDEX_PARSER = new ObjectParser<>("index", Builder::new); static { INDEX_PARSER.declareString(Builder::name, new ParseField(INDEX_NAME_KEY)); @@ -66,39 +69,74 @@ public class Index implements Writeable, ToXContentObject { private final String name; private final String uuid; + /** + * Creates a new Index instance with name and unique identifier + * + * @param name the name of the index + * @param uuid the unique identifier of the index + * @throws NullPointerException if either name or uuid are null + */ public Index(String name, String uuid) { this.name = Objects.requireNonNull(name); this.uuid = Objects.requireNonNull(uuid); } /** - * Read from a stream. + * Creates a new Index instance from a {@link StreamInput}. + * Reads the name and unique identifier from the stream. + * + * @param in the stream to read from + * @throws IOException if an error occurs while reading from the stream + * @see #writeTo(StreamOutput) */ public Index(StreamInput in) throws IOException { this.name = in.readString(); this.uuid = in.readString(); } + /** + * Gets the name of the index. + * + * @return the name of the index. + */ public String getName() { return this.name; } + /** + * Gets the unique identifier of the index. + * + * @return the unique identifier of the index. "_na_" if {@link Strings#UNKNOWN_UUID_VALUE}. + */ public String getUUID() { return uuid; } + /** + * Returns either the name and unique identifier of the index + * or only the name if the uuid is {@link Strings#UNKNOWN_UUID_VALUE}. + * + * If we have a uuid we put it in the toString so it'll show up in logs + * which is useful as more and more things use the uuid rather + * than the name as the lookup key for the index. + * + * @return {@code "[name/uuid]"} or {@code "[name]"} + */ @Override public String toString() { - /* - * If we have a uuid we put it in the toString so it'll show up in logs which is useful as more and more things use the uuid rather - * than the name as the lookup key for the index. - */ if (Strings.UNKNOWN_UUID_VALUE.equals(uuid)) { return "[" + name + "]"; } return "[" + name + "/" + uuid + "]"; } + /** + * Checks if this index is the same as another index by comparing the name and unique identifier. + * If both uuid are {@link Strings#UNKNOWN_UUID_VALUE} then only the name is compared. + * + * @param o the index to compare to + * @return true if the name and unique identifier are the same, false otherwise. + */ @Override public boolean equals(Object o) { if (this == o) { @@ -118,6 +156,10 @@ public int hashCode() { return result; } + /** Writes the name and unique identifier to the {@link StreamOutput} + * + * @param out The stream to write to + */ @Override public void writeTo(final StreamOutput out) throws IOException { out.writeString(name); diff --git a/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java b/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java index adea6cd8f0687..984434190b486 100644 --- a/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java +++ b/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java @@ -55,44 +55,87 @@ public class ShardId implements Comparable, ToXContentFragment, Writeab private final int shardId; private final int hashCode; + /** + * Constructs a new shard id. + * @param index the index name + * @param shardId the shard id + */ public ShardId(Index index, int shardId) { this.index = index; this.shardId = shardId; this.hashCode = computeHashCode(); } + /** + * Constructs a new shard id with the given index name, index unique identifier, and shard id. + * @param index the index name + * @param indexUUID the index unique identifier + * @param shardId the shard id + */ public ShardId(String index, String indexUUID, int shardId) { this(new Index(index, indexUUID), shardId); } + /** + * Constructs a new shardId from a stream. + * @param in the stream to read from + * @throws IOException if an error occurs while reading from the stream + * @see #writeTo(StreamOutput) + */ public ShardId(StreamInput in) throws IOException { index = new Index(in); shardId = in.readVInt(); hashCode = computeHashCode(); } + /** + * Writes this shard id to a stream. + * @param out the stream to write to + * @throws IOException if an error occurs while writing to the stream + */ @Override public void writeTo(StreamOutput out) throws IOException { index.writeTo(out); out.writeVInt(shardId); } + /** + * Returns the index of this shard id. + * @return the index of this shard id + */ public Index getIndex() { return index; } + /** + * Returns the name of the index of this shard id. + * @return the name of the index of this shard id + */ public String getIndexName() { return index.getName(); } + /** + * Return the shardId of this shard id. + * @return the shardId of this shard id + * @see #getId() + */ public int id() { return this.shardId; } + /** + * Returns the shard id of this shard id. + * @return the shard id of this shard id + */ public int getId() { return id(); } + /** + * Returns a string representation of this shard id. + * @return "[indexName][shardId]" + */ @Override public String toString() { return "[" + index.getName() + "][" + shardId + "]"; @@ -100,9 +143,13 @@ public String toString() { /** * Parse the string representation of this shardId back to an object. + * * We lose index uuid information here, but since we use toString in * rest responses, this is the best we can do to reconstruct the object * on the client side. + * + * @param shardIdString the string representation of the shard id + * (Expect a string of format "[indexName][shardId]" (square brackets included)) */ public static ShardId fromString(String shardIdString) { int splitPosition = shardIdString.indexOf("]["); @@ -122,17 +169,30 @@ public boolean equals(Object o) { return shardId == shardId1.shardId && index.equals(shardId1.index); } + /** Returns the hash code of this shard id. + * + * @return the hash code of this shard id + */ @Override public int hashCode() { return hashCode; } + /** Computes the hash code of this shard id. + * + * @return the hash code of this shard id. + */ private int computeHashCode() { int result = index != null ? index.hashCode() : 0; result = 31 * result + shardId; return result; } + /** + * Compares this ShardId with the specified ShardId. + * @param o the ShardId to be compared. + * @return a negative integer, zero, or a positive integer if this ShardId is less than, equal to, or greater than the specified ShardId + */ @Override public int compareTo(ShardId o) { if (o.getId() == shardId) { diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java index ab887acb85a87..3ce8b4953b9d6 100644 --- a/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java @@ -47,25 +47,51 @@ */ public class AllCircuitBreakerStats implements Writeable, ToXContentFragment { + /** An array of all the circuit breaker stats */ private final CircuitBreakerStats[] allStats; + /** + * Constructs the instance + * + * @param allStats an array of all the circuit breaker stats + */ public AllCircuitBreakerStats(CircuitBreakerStats[] allStats) { this.allStats = allStats; } + /** + * Constructs the new instance from {@link StreamInput} + * @param in the {@link StreamInput} to read from + * @throws IOException If an error occurs while reading from the StreamInput + * @see #writeTo(StreamOutput) + */ public AllCircuitBreakerStats(StreamInput in) throws IOException { allStats = in.readArray(CircuitBreakerStats::new, CircuitBreakerStats[]::new); } + /** + * Writes this instance into a {@link StreamOutput} + * @param out the {@link StreamOutput} to write to + * @throws IOException if an error occurs while writing to the StreamOutput + */ @Override public void writeTo(StreamOutput out) throws IOException { out.writeArray(allStats); } + /** + * Returns inner stats instances for all circuit breakers + * @return inner stats instances for all circuit breakers + */ public CircuitBreakerStats[] getAllStats() { return this.allStats; } + /** + * Returns the stats for a specific circuit breaker + * @param name the name of the circuit breaker + * @return the {@link CircuitBreakerStats} for the circuit breaker, null if the circuit breaker with such name does not exist + */ public CircuitBreakerStats getStats(String name) { for (CircuitBreakerStats stats : allStats) { if (stats.getName().equals(name)) { diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java index 0e53a38908a96..9207d3ea77227 100644 --- a/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java @@ -43,18 +43,33 @@ import java.util.Locale; /** - * Class encapsulating stats about the circuit breaker + * Class encapsulating stats about the {@link org.opensearch.core.common.breaker.CircuitBreaker} * * @opensearch.internal */ public class CircuitBreakerStats implements Writeable, ToXContentObject { + /** The name of the circuit breaker */ private final String name; + /** The limit size in byte of the circuit breaker. Field : "limit_size_in_bytes" */ private final long limit; + /** The estimated size in byte of the breaker. Field : "estimated_size_in_bytes" */ private final long estimated; + /** The number of times the breaker has been tripped. Field : "tripped" */ private final long trippedCount; + /** The overhead of the breaker. Field : "overhead" */ private final double overhead; + /** + * Constructs new instance + * + * @param name The name of the circuit breaker + * @param limit The limit size in byte of the circuit breaker + * @param estimated The estimated size in byte of the breaker + * @param overhead The overhead of the breaker + * @param trippedCount The number of times the breaker has been tripped + * @see org.opensearch.core.common.breaker.CircuitBreaker + */ public CircuitBreakerStats(String name, long limit, long estimated, double overhead, long trippedCount) { this.name = name; this.limit = limit; @@ -63,6 +78,14 @@ public CircuitBreakerStats(String name, long limit, long estimated, double overh this.overhead = overhead; } + /** + * Constructs new instance from the {@link StreamInput} + * + * @param in The StreamInput + * @throws IOException if an error occurs while reading from the StreamInput + * @see org.opensearch.core.common.breaker.CircuitBreaker + * @see #writeTo(StreamOutput) + */ public CircuitBreakerStats(StreamInput in) throws IOException { this.limit = in.readLong(); this.estimated = in.readLong(); @@ -71,6 +94,13 @@ public CircuitBreakerStats(StreamInput in) throws IOException { this.name = in.readString(); } + /** + * Writes this instance into a {@link StreamOutput} + * + * @param out The StreamOutput + * @throws IOException if an error occurs while writing to the StreamOutput + * @see #CircuitBreakerStats(StreamInput) + */ @Override public void writeTo(StreamOutput out) throws IOException { out.writeLong(limit); @@ -80,22 +110,42 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(name); } + /** + * Returns the name of the circuit breaker + * @return The name of the circuit breaker + */ public String getName() { return this.name; } + /** + * Returns the limit size in byte of the circuit breaker + * @return The limit size in byte of the circuit breaker + */ public long getLimit() { return this.limit; } + /** + * Returns the estimated size in byte of the breaker + * @return The estimated size in byte of the breaker + */ public long getEstimated() { return this.estimated; } + /** + * Returns the number of times the breaker has been tripped + * @return The number of times the breaker has been tripped + */ public long getTrippedCount() { return this.trippedCount; } + /** + * Returns the overhead of the breaker + * @return The overhead of the breaker + */ public double getOverhead() { return this.overhead; } @@ -113,6 +163,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + /** + * Returns a String representation of this CircuitBreakerStats + * @return "[name,limit=limit/limit_human,estimated=estimated/estimated_human,overhead=overhead,tripped=trippedCount]" + */ @Override public String toString() { return "[" diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java index 4095fd32b6d3c..49c5a393328b9 100644 --- a/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java @@ -36,8 +36,9 @@ import org.opensearch.core.common.breaker.NoopCircuitBreaker; /** - * Class that returns a breaker that never breaks + * Class that returns a breaker that use the NoopCircuitBreaker and never breaks * + * @see org.opensearch.core.common.breaker.NoopCircuitBreaker * @opensearch.internal */ public class NoneCircuitBreakerService extends CircuitBreakerService { @@ -48,6 +49,12 @@ public NoneCircuitBreakerService() { super(); } + /** + * Returns a breaker that use the NoopCircuitBreaker and never breaks + * + * @param name name of the breaker (ignored) + * @return a NoopCircuitBreaker + */ @Override public CircuitBreaker getBreaker(String name) { return breaker; @@ -58,6 +65,12 @@ public AllCircuitBreakerStats stats() { return new AllCircuitBreakerStats(new CircuitBreakerStats[] { stats(CircuitBreaker.FIELDDATA) }); } + /** + * Always returns the same stats, a NoopCircuitBreaker never breaks and all operations are noops. + * + * @param name name of the breaker (ignored) + * @return always "fielddata", limit: -1, estimated: -1, overhead: 0, trippedCount: 0 + */ @Override public CircuitBreakerStats stats(String name) { return new CircuitBreakerStats(CircuitBreaker.FIELDDATA, -1, -1, 0, 0); diff --git a/libs/geo/src/main/java/org/opensearch/geometry/Circle.java b/libs/geo/src/main/java/org/opensearch/geometry/Circle.java index 6f8b0dc6929cc..cf1dce8966e1f 100644 --- a/libs/geo/src/main/java/org/opensearch/geometry/Circle.java +++ b/libs/geo/src/main/java/org/opensearch/geometry/Circle.java @@ -39,12 +39,19 @@ * and optional altitude in meters. */ public class Circle implements Geometry { + + /** Empty circle : x=0, y=0, z=NaN radius=-1 */ public static final Circle EMPTY = new Circle(); + /** Latitude of the center of the circle in degrees */ private final double y; + /** Longitude of the center of the circle in degrees */ private final double x; + /** Altitude of the center of the circle in meters (NaN if irrelevant) */ private final double z; + /** Radius of the circle in meters */ private final double radiusMeters; + /** Create an {@link #EMPTY} circle */ private Circle() { y = 0; x = 0; @@ -52,10 +59,23 @@ private Circle() { radiusMeters = -1; } + /** + * Create a circle with no altitude. + * @param x Longitude of the center of the circle in degrees + * @param y Latitude of the center of the circle in degrees + * @param radiusMeters Radius of the circle in meters + */ public Circle(final double x, final double y, final double radiusMeters) { this(x, y, Double.NaN, radiusMeters); } + /** + * Create a circle with altitude. + * @param x Longitude of the center of the circle in degrees + * @param y Latitude of the center of the circle in degrees + * @param z Altitude of the center of the circle in meters + * @param radiusMeters Radius of the circle in meters + */ public Circle(final double x, final double y, final double z, final double radiusMeters) { this.y = y; this.x = x; @@ -66,39 +86,68 @@ public Circle(final double x, final double y, final double z, final double radiu } } + /** + * @return The type of this geometry (always {@link ShapeType#CIRCLE}) + */ @Override public ShapeType type() { return ShapeType.CIRCLE; } + /** + * @return The y (latitude) of the center of the circle in degrees + */ public double getY() { return y; } + /** + * @return The x (longitude) of the center of the circle in degrees + */ public double getX() { return x; } + /** + * @return The radius of the circle in meters + */ public double getRadiusMeters() { return radiusMeters; } + /** + * @return The altitude of the center of the circle in meters (NaN if irrelevant) + */ public double getZ() { return z; } + /** + * @return The latitude (y) of the center of the circle in degrees + */ public double getLat() { return y; } + /** + * @return The longitude (x) of the center of the circle in degrees + */ public double getLon() { return x; } + /** + * @return The altitude (z) of the center of the circle in meters (NaN if irrelevant) + */ public double getAlt() { return z; } + /** + * Compare this circle to another circle. + * @param o The other circle + * @return True if the two circles are equal in all their properties. False if null or different. + */ @Override public boolean equals(Object o) { if (this == o) return true; @@ -111,6 +160,9 @@ public boolean equals(Object o) { return (Double.compare(circle.z, z) == 0); } + /** + * @return The hashcode of this circle. + */ @Override public int hashCode() { int result; @@ -126,11 +178,22 @@ public int hashCode() { return result; } + /** + * Visit this circle with a {@link GeometryVisitor}. + * @param visitor The visitor + * @param The return type of the visitor + * @param The exception type of the visitor + * @return The result of the visitor + * @throws E The exception thrown by the visitor + */ @Override public T visit(GeometryVisitor visitor) throws E { return visitor.visit(this); } + /** + * @return True if this circle is empty (radius less than 0) + */ @Override public boolean isEmpty() { return radiusMeters < 0; @@ -141,6 +204,9 @@ public String toString() { return WellKnownText.INSTANCE.toWKT(this); } + /** + * @return True if this circle has an altitude. False if NaN. + */ @Override public boolean hasZ() { return Double.isNaN(z) == false; From e6dec2906f07753cc93030551694864aef4e3f93 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 14 Sep 2023 15:46:58 -0700 Subject: [PATCH 031/388] Disable stalled PR workflow (#10052) Signed-off-by: Kunal Kotwani --- .github/workflows/stalled.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/stalled.yml diff --git a/.github/workflows/stalled.yml b/.github/workflows/stalled.yml deleted file mode 100644 index bc0a98fff511e..0000000000000 --- a/.github/workflows/stalled.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Close Stalled PRs -on: - schedule: - - cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT -permissions: - pull-requests: write -jobs: - stale: - if: github.repository == 'opensearch-project/OpenSearch' - runs-on: ubuntu-latest - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.5.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Stale PRs - uses: actions/stale@v8 - with: - repo-token: ${{ steps.github_app_token.outputs.token }} - stale-pr-label: 'stalled' - stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity. Remove stalled label or comment or this will be closed in 7 days.' - close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity.' - days-before-pr-stale: 30 - days-before-pr-close: 7 - days-before-issue-stale: -1 - days-before-issue-close: -1 From d77fb2489dfe2202a6bf8f0c82c8a5cf453c8262 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 14 Sep 2023 16:28:01 -0700 Subject: [PATCH 032/388] Add release notes for 1.3.13 (#10058) Signed-off-by: Kunal Kotwani --- release-notes/opensearch.release-notes-1.3.13.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 release-notes/opensearch.release-notes-1.3.13.md diff --git a/release-notes/opensearch.release-notes-1.3.13.md b/release-notes/opensearch.release-notes-1.3.13.md new file mode 100644 index 0000000000000..3ece2c8f91984 --- /dev/null +++ b/release-notes/opensearch.release-notes-1.3.13.md @@ -0,0 +1,7 @@ +## 2023-09-14 Version 1.3.13 Release Notes + +### Upgrades +- Bump `netty` from 4.1.96.Final to 4.1.97.Final ([#9553](https://github.com/opensearch-project/OpenSearch/pull/9553)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.8.2 to 1.1.10.3 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) +- Bump `com.squareup.okhttp3:okhttp` from 4.9.3 to 4.11.0 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) +- Bump `com.squareup.okio:okio` from 2.8.0 to 3.5.0 ([#9252](https://github.com/opensearch-project/OpenSearch/pull/9252)) From 7d1fa7d11d32bf8b03f9f8641ed7f147b6eb2e34 Mon Sep 17 00:00:00 2001 From: Harish Bhakuni Date: Thu, 14 Sep 2023 18:14:42 -0700 Subject: [PATCH 033/388] Parameterize ITs to run with concurrent search enabled (#9441) * Parameterize ITs to run with concurrent search enabled Signed-off-by: Harish Bhakuni * Address PR Changes Signed-off-by: Harish Bhakuni * Address PR Comments Signed-off-by: Harish Bhakuni --------- Signed-off-by: Harish Bhakuni Co-authored-by: Harish Bhakuni --- .../indices/IndicesRequestCacheIT.java | 25 ++++++++++++++-- .../indices/stats/IndexStatsIT.java | 24 +++++++++++++-- .../org/opensearch/script/ScriptCacheIT.java | 25 ++++++++++++++-- .../search/StressSearchServiceReaperIT.java | 26 ++++++++++++++-- .../opensearch/search/pit/PitMultiNodeIT.java | 25 +++++++++++++++- .../scriptfilter/ScriptQuerySearchIT.java | 23 +++++++++++++- .../search/scroll/DuelScrollIT.java | 25 ++++++++++++++-- .../search/scroll/SearchScrollIT.java | 27 +++++++++++++++-- .../SearchScrollWithFailingNodesIT.java | 30 ++++++++++++++++--- .../search/searchafter/SearchAfterIT.java | 27 +++++++++++++++-- .../search/sort/GeoDistanceSortBuilderIT.java | 25 ++++++++++++++-- .../search/sort/SortFromPluginIT.java | 25 ++++++++++++++-- .../suggest/CompletionSuggestSearchIT.java | 24 +++++++++++++-- .../ContextCompletionSuggestSearchIT.java | 24 +++++++++++++-- .../search/suggest/SuggestSearchIT.java | 24 +++++++++++++-- .../opensearch/similarity/SimilarityIT.java | 28 +++++++++++++++-- 16 files changed, 375 insertions(+), 32 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index 12fee85288bc2..98a22717019cf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -32,6 +32,8 @@ package org.opensearch.indices; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.opensearch.action.search.SearchResponse; @@ -40,13 +42,14 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.cache.request.RequestCacheStats; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.time.ZoneId; @@ -54,8 +57,10 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Arrays; +import java.util.Collection; import java.util.List; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.filter; @@ -64,7 +69,23 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -public class IndicesRequestCacheIT extends OpenSearchIntegTestCase { +public class IndicesRequestCacheIT extends ParameterizedOpenSearchIntegTestCase { + public IndicesRequestCacheIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } // One of the primary purposes of the query cache is to cache aggs results public void testCacheAggs() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java index ee904dbcb6924..a0f01acd1f8e9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java @@ -32,6 +32,8 @@ package org.opensearch.indices.stats; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -55,6 +57,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamOutput; @@ -80,9 +83,9 @@ import org.opensearch.plugins.Plugin; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.InternalSettingsPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; @@ -105,6 +108,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; @@ -118,7 +122,23 @@ @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0) @SuppressCodecs("*") // requires custom completion format -public class IndexStatsIT extends OpenSearchIntegTestCase { +public class IndexStatsIT extends ParameterizedOpenSearchIntegTestCase { + public IndexStatsIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java index e125492d4b2c5..2fbaf4ea5a4d3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/script/ScriptCacheIT.java @@ -8,8 +8,11 @@ package org.opensearch.script; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.MockEngineFactoryPlugin; @@ -18,12 +21,13 @@ import org.opensearch.plugins.Plugin; import org.opensearch.search.MockSearchService; import org.opensearch.test.MockHttpTransport; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.TestGeoShapeFieldMapperPlugin; import org.opensearch.test.store.MockFSIndexStore; import org.opensearch.test.transport.MockTransportService; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -31,9 +35,26 @@ import java.util.concurrent.ExecutionException; import java.util.function.Function; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.apache.logging.log4j.core.util.Throwables.getRootCause; -public class ScriptCacheIT extends OpenSearchIntegTestCase { +public class ScriptCacheIT extends ParameterizedOpenSearchIntegTestCase { + public ScriptCacheIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/StressSearchServiceReaperIT.java b/server/src/internalClusterTest/java/org/opensearch/search/StressSearchServiceReaperIT.java index 42e515cca9b6b..a61102b9db144 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/StressSearchServiceReaperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/StressSearchServiceReaperIT.java @@ -31,23 +31,45 @@ package org.opensearch.search; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.tests.util.English; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; +import java.util.Arrays; +import java.util.Collection; import java.util.concurrent.ExecutionException; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.OpenSearchIntegTestCase.Scope.SUITE; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @ClusterScope(scope = SUITE) -public class StressSearchServiceReaperIT extends OpenSearchIntegTestCase { +public class StressSearchServiceReaperIT extends ParameterizedOpenSearchIntegTestCase { + public StressSearchServiceReaperIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java index 61a5f76a32979..e42f12709c948 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java @@ -8,6 +8,8 @@ package org.opensearch.search.pit; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.LatchedActionListener; import org.opensearch.action.admin.cluster.state.ClusterStateRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; @@ -30,10 +32,12 @@ import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.ActionListener; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; import org.junit.After; @@ -41,6 +45,8 @@ import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -53,6 +59,7 @@ import static org.opensearch.action.search.PitTestsUtil.assertSegments; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; @@ -60,7 +67,23 @@ * Multi node integration tests for PIT creation and search operation with PIT ID. */ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2) -public class PitMultiNodeIT extends OpenSearchIntegTestCase { +public class PitMultiNodeIT extends ParameterizedOpenSearchIntegTestCase { + public PitMultiNodeIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Before public void setupIndex() throws ExecutionException, InterruptedException { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java index e081be0af51a2..34967528f2c4f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java @@ -32,10 +32,13 @@ package org.opensearch.search.scriptfilter; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexModule; @@ -47,6 +50,7 @@ import org.opensearch.search.sort.SortOrder; import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.Arrays; @@ -61,12 +65,29 @@ import static java.util.Collections.emptyMap; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.scriptQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE) -public class ScriptQuerySearchIT extends OpenSearchIntegTestCase { +public class ScriptQuerySearchIT extends ParameterizedOpenSearchIntegTestCase { + public ScriptQuerySearchIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java index e0a54e9b4fc36..c7a6d18f881c6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/DuelScrollIT.java @@ -32,6 +32,7 @@ package org.opensearch.search.scroll; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.opensearch.action.index.IndexRequestBuilder; @@ -39,24 +40,44 @@ import org.opensearch.action.search.SearchType; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.Set; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; -public class DuelScrollIT extends OpenSearchIntegTestCase { +public class DuelScrollIT extends ParameterizedOpenSearchIntegTestCase { + public DuelScrollIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testDuelQueryThenFetch() throws Exception { TestContext context = create(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java index aec6a03d3e57f..0eee136acac69 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.scroll; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.ExceptionsHelper; import org.opensearch.action.search.ClearScrollResponse; import org.opensearch.action.search.SearchPhaseExecutionException; @@ -43,6 +45,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesReference; @@ -57,11 +60,13 @@ import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.InternalTestCluster; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; import org.junit.After; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Map; @@ -70,6 +75,7 @@ import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.queryStringQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @@ -86,7 +92,24 @@ /** * Tests for scrolling. */ -public class SearchScrollIT extends OpenSearchIntegTestCase { +public class SearchScrollIT extends ParameterizedOpenSearchIntegTestCase { + public SearchScrollIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @After public void cleanup() throws Exception { assertAcked( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java index c6519cc3a0cb3..f16b9a4d67b49 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java @@ -32,26 +32,50 @@ package org.opensearch.search.scroll; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) -public class SearchScrollWithFailingNodesIT extends OpenSearchIntegTestCase { +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 2, numClientNodes = 0) +public class SearchScrollWithFailingNodesIT extends ParameterizedOpenSearchIntegTestCase { + public SearchScrollWithFailingNodesIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + @Override protected int numberOfShards() { return 2; @@ -63,8 +87,6 @@ protected int numberOfReplicas() { } public void testScanScrollWithShardExceptions() throws Exception { - internalCluster().startNode(); - internalCluster().startNode(); assertAcked( prepareCreate("test") // Enforces that only one shard can only be allocated to a single node diff --git a/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java index 22c0a9cbbab17..00ac574b8bd72 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.searchafter; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.CreatePitAction; @@ -43,30 +45,51 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.common.UUIDs; import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.hamcrest.Matchers; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -public class SearchAfterIT extends OpenSearchIntegTestCase { +public class SearchAfterIT extends ParameterizedOpenSearchIntegTestCase { private static final String INDEX_NAME = "test"; private static final int NUM_DOCS = 100; + public SearchAfterIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testsShouldFail() throws Exception { assertAcked(client().admin().indices().prepareCreate("test").setMapping("field1", "type=long", "field2", "type=keyword").get()); ensureGreen(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java index 5a0ca1d13633e..7880fc24fd846 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.sort; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.Version; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.IndexMetadata; @@ -39,28 +41,47 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.DistanceUnit; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.GeoValidationMethod; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.sort.SortBuilders.fieldSort; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertOrderedSearchHits; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSortValues; import static org.hamcrest.Matchers.closeTo; -public class GeoDistanceSortBuilderIT extends OpenSearchIntegTestCase { +public class GeoDistanceSortBuilderIT extends ParameterizedOpenSearchIntegTestCase { + public GeoDistanceSortBuilderIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } private static final String LOCATION_FIELD = "location"; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java index 5b896f9a1fe57..7bcded86fcaa8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java @@ -8,21 +8,42 @@ package org.opensearch.search.sort; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.plugins.Plugin; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.plugin.CustomSortBuilder; import org.opensearch.search.sort.plugin.CustomSortPlugin; import org.opensearch.test.InternalSettingsPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.Arrays; import java.util.Collection; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.hamcrest.Matchers.equalTo; -public class SortFromPluginIT extends OpenSearchIntegTestCase { +public class SortFromPluginIT extends ParameterizedOpenSearchIntegTestCase { + public SortFromPluginIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } @Override protected Collection> nodePlugins() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java index 7183f18acbadf..30dba87f8ef5d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java @@ -31,6 +31,7 @@ package org.opensearch.search.suggest; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.apache.lucene.analysis.TokenStreamToAutomaton; @@ -47,6 +48,7 @@ import org.opensearch.common.FieldMemoryStats; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperParsingException; @@ -63,7 +65,7 @@ import org.opensearch.search.suggest.completion.context.ContextMapping; import org.opensearch.search.suggest.completion.context.GeoContextMapping; import org.opensearch.test.InternalSettingsPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; @@ -81,6 +83,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.core.common.util.CollectionUtils.iterableAsArrayList; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.opensearch.test.hamcrest.OpenSearchAssertions.hasId; @@ -96,7 +99,24 @@ import static org.hamcrest.Matchers.notNullValue; @SuppressCodecs("*") // requires custom completion format -public class CompletionSuggestSearchIT extends OpenSearchIntegTestCase { +public class CompletionSuggestSearchIT extends ParameterizedOpenSearchIntegTestCase { + public CompletionSuggestSearchIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + private final String INDEX = RandomStrings.randomAsciiOfLength(random(), 10).toLowerCase(Locale.ROOT); private final String FIELD = RandomStrings.randomAsciiOfLength(random(), 10).toLowerCase(Locale.ROOT); private final CompletionMappingBuilder completionMappingBuilder = new CompletionMappingBuilder(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java index 7f5e8abfc3b52..bac3e7fb61683 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -31,6 +31,7 @@ package org.opensearch.search.suggest; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; @@ -40,6 +41,7 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -51,11 +53,12 @@ import org.opensearch.search.suggest.completion.context.ContextMapping; import org.opensearch.search.suggest.completion.context.GeoContextMapping; import org.opensearch.search.suggest.completion.context.GeoQueryContext; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -64,12 +67,29 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; import static org.hamcrest.core.IsEqual.equalTo; @SuppressCodecs("*") // requires custom completion format -public class ContextCompletionSuggestSearchIT extends OpenSearchIntegTestCase { +public class ContextCompletionSuggestSearchIT extends ParameterizedOpenSearchIntegTestCase { + public ContextCompletionSuggestSearchIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } private final String INDEX = RandomStrings.randomAsciiOfLength(random(), 10).toLowerCase(Locale.ROOT); private final String FIELD = RandomStrings.randomAsciiOfLength(random(), 10).toLowerCase(Locale.ROOT); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java index 017dd5ea668de..32bb0e34054bb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java @@ -32,6 +32,8 @@ package org.opensearch.search.suggest; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.OpenSearchException; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.action.index.IndexRequestBuilder; @@ -39,6 +41,7 @@ import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; @@ -54,7 +57,7 @@ import org.opensearch.search.suggest.phrase.StupidBackoff; import org.opensearch.search.suggest.term.TermSuggestionBuilder; import org.opensearch.search.suggest.term.TermSuggestionBuilder.SuggestMode; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.io.IOException; @@ -73,6 +76,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.index.query.QueryBuilders.matchQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.suggest.SuggestBuilders.phraseSuggestion; import static org.opensearch.search.suggest.SuggestBuilders.termSuggestion; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -92,7 +96,23 @@ * possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that * request, modify again, request again, etc. This makes it very obvious what changes between requests. */ -public class SuggestSearchIT extends OpenSearchIntegTestCase { +public class SuggestSearchIT extends ParameterizedOpenSearchIntegTestCase { + public SuggestSearchIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } // see #3196 public void testSuggestAcrossMultipleIndices() throws IOException { diff --git a/server/src/internalClusterTest/java/org/opensearch/similarity/SimilarityIT.java b/server/src/internalClusterTest/java/org/opensearch/similarity/SimilarityIT.java index 929aac388b678..8c9bff9833462 100644 --- a/server/src/internalClusterTest/java/org/opensearch/similarity/SimilarityIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/similarity/SimilarityIT.java @@ -32,17 +32,41 @@ package org.opensearch.similarity; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; -public class SimilarityIT extends OpenSearchIntegTestCase { +public class SimilarityIT extends ParameterizedOpenSearchIntegTestCase { + public SimilarityIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + public void testCustomBM25Similarity() throws Exception { try { client().admin().indices().prepareDelete("test").execute().actionGet(); From 74fcaab442ba6dec314ac31615f814909bc78297 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 15 Sep 2023 16:14:58 +0530 Subject: [PATCH 034/388] Fix flaky behavior of RemoteStoreRestoreIT.testRateLimitedRemoteDownloads (#9887) * Fix flaky behavior of RemoteStoreRestoreIT.testRateLimitedRemoteDownloads Signed-off-by: Sachin Kale --- .../opensearch/remotestore/RemoteStoreBaseIntegTestCase.java | 2 +- .../org/opensearch/remotestore/RemoteStoreRestoreIT.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 9a684ce0a1482..621fb262c0a91 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -230,7 +230,7 @@ public static Settings buildRemoteStoreNodeAttributes( if (withRateLimiterAttributes) { settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) - .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "2kb") + .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "4kb") .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 489f4c52d4298..65335f444a2df 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -406,9 +406,10 @@ public void testRateLimitedRemoteDownloads() throws Exception { for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) { downloadPauseTime += repositoriesService.repository(REPOSITORY_NAME).getRemoteDownloadThrottleTimeInNanos(); } - assertThat(downloadPauseTime, greaterThan(TimeValue.timeValueSeconds(randomIntBetween(5, 10)).nanos())); + assertThat(downloadPauseTime, greaterThan(TimeValue.timeValueSeconds(randomIntBetween(3, 5)).nanos())); }, 30, TimeUnit.SECONDS); - ensureGreen(INDEX_NAME); + // Waiting for extended period for green state so that rate limit does not cause flakiness + ensureGreen(TimeValue.timeValueSeconds(120), INDEX_NAME); // This is required to get updated number from already active shards which were not restored assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); assertEquals(0, getNumShards(INDEX_NAME).numReplicas); From 483c2e36f6c8f2e326279b3b0e598d21072bdc14 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Fri, 15 Sep 2023 21:15:24 +0530 Subject: [PATCH 035/388] Add tracing instrumentation in transport service (#10042) * Add instrumentation to transport service Signed-off-by: Gagan Juneja * Add instrumentation to transport service Signed-off-by: Gagan Juneja * Add javadoc Signed-off-by: Gagan Juneja --------- Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- CHANGELOG.md | 1 + .../TelemetryTracerEnabledSanityIT.java | 2 +- .../single/SingleNodeDiscoveryIT.java | 2 + .../TraceableTransportResponseHandler.java | 107 ++++++++++++++++++ .../tracing/handler/package-info.java | 12 ++ .../transport/TransportResponseHandler.java | 7 ++ .../transport/TransportService.java | 99 +++++++++------- .../test/OpenSearchIntegTestCase.java | 3 +- 8 files changed, 188 insertions(+), 45 deletions(-) create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/handler/package-info.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a52c3d7f2a3f..08f3c3abd4162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) - Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916)) - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) +- Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042)) ### Deprecated diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java index 2d0111e64faad..8a49a0abf5512 100644 --- a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java +++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java @@ -89,7 +89,7 @@ public void testSanityChecksWhenTracingEnabled() throws Exception { InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.INSTANCE; if (!exporter.getFinishedSpanItems().isEmpty()) { - validators.validate(exporter.getFinishedSpanItems(), 2); + validators.validate(exporter.getFinishedSpanItems(), 6); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/single/SingleNodeDiscoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/single/SingleNodeDiscoveryIT.java index 90bdcf7fded11..1f6c8eac6c391 100644 --- a/server/src/internalClusterTest/java/org/opensearch/discovery/single/SingleNodeDiscoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/discovery/single/SingleNodeDiscoveryIT.java @@ -76,6 +76,7 @@ public void testSingleNodesDoNotDiscoverEachOther() throws IOException, Interrup @Override public Settings nodeSettings(int nodeOrdinal) { return Settings.builder() + .put(featureFlagSettings()) .put("discovery.type", "single-node") .put("transport.type", getTestTransportType()) /* @@ -142,6 +143,7 @@ public boolean innerMatch(final LogEvent event) { @Override public Settings nodeSettings(int nodeOrdinal) { return Settings.builder() + .put(featureFlagSettings()) .put("discovery.type", "zen") .put("transport.type", getTestTransportType()) .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java new file mode 100644 index 0000000000000..abddfcc6cebc1 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.handler; + +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.transport.TransportException; +import org.opensearch.transport.TransportResponseHandler; + +import java.io.IOException; +import java.util.Objects; + +/** + * Tracer wrapped {@link TransportResponseHandler} + * @param TransportResponse + */ +public class TraceableTransportResponseHandler implements TransportResponseHandler { + + private final Span span; + private final TransportResponseHandler delegate; + private final Tracer tracer; + + /** + * Constructor. + * + * @param delegate delegate + * @param span span + * @param tracer tracer + */ + private TraceableTransportResponseHandler(TransportResponseHandler delegate, Span span, Tracer tracer) { + this.delegate = Objects.requireNonNull(delegate); + this.span = Objects.requireNonNull(span); + this.tracer = Objects.requireNonNull(tracer); + } + + /** + * Factory method. + * @param delegate delegate + * @param span span + * @param tracer tracer + * @return transportResponseHandler + */ + public static TransportResponseHandler create( + TransportResponseHandler delegate, + Span span, + Tracer tracer + ) { + if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) { + return new TraceableTransportResponseHandler(delegate, span, tracer); + } else { + return delegate; + } + } + + @Override + public T read(StreamInput in) throws IOException { + return delegate.read(in); + } + + @Override + public void handleResponse(T response) { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.handleResponse(response); + } finally { + span.endSpan(); + } + } + + @Override + public void handleException(TransportException exp) { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.handleException(exp); + } finally { + span.setError(exp); + span.endSpan(); + } + } + + @Override + public String executor() { + return delegate.executor(); + } + + @Override + public String toString() { + return delegate.toString(); + } + + @Override + public void handleRejection(Exception exp) { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.handleRejection(exp); + } finally { + span.endSpan(); + } + } +} diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/package-info.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/package-info.java new file mode 100644 index 0000000000000..ff9f8f57dc07c --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for tracing requests. + */ +package org.opensearch.telemetry.tracing.handler; diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java index 0b39983cc3bee..90e94e52515ce 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java @@ -52,6 +52,13 @@ public interface TransportResponseHandler extends W String executor(); + /** + * This method should be handling the rejection/failure scenarios where connection to the node is rejected or failed. + * It should be used to clear up the resources held by the {@link TransportResponseHandler}. + * @param exp exception + */ + default void handleRejection(Exception exp) {}; + default TransportResponseHandler wrap(Function converter, Writeable.Reader reader) { final TransportResponseHandler self = this; return new TransportResponseHandler() { diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index 52274872e8cc8..32bedb52a9cef 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -66,7 +66,11 @@ import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanBuilder; +import org.opensearch.telemetry.tracing.SpanScope; import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.telemetry.tracing.handler.TraceableTransportResponseHandler; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -333,6 +337,7 @@ protected void doStop() { getExecutorService().execute(new AbstractRunnable() { @Override public void onRejection(Exception e) { + holderToNotify.handler().handleRejection(e); // if we get rejected during node shutdown we don't wanna bubble it up logger.debug( () -> new ParameterizedMessage( @@ -345,6 +350,7 @@ public void onRejection(Exception e) { @Override public void onFailure(Exception e) { + holderToNotify.handler().handleRejection(e); logger.warn( () -> new ParameterizedMessage( "failed to notify response handler on exception, action: {}", @@ -861,53 +867,60 @@ public final void sendRequest( final TransportRequestOptions options, final TransportResponseHandler handler ) { - try { - logger.debug("Action: " + action); - final TransportResponseHandler delegate; - if (request.getParentTask().isSet()) { - // TODO: capture the connection instead so that we can cancel child tasks on the remote connections. - final Releasable unregisterChildNode = taskManager.registerChildNode(request.getParentTask().getId(), connection.getNode()); - delegate = new TransportResponseHandler() { - @Override - public void handleResponse(T response) { - unregisterChildNode.close(); - handler.handleResponse(response); - } + final Span span = tracer.startSpan(SpanBuilder.from(action, connection)); + try (SpanScope spanScope = tracer.withSpanInScope(span)) { + TransportResponseHandler traceableTransportResponseHandler = TraceableTransportResponseHandler.create(handler, span, tracer); + try { + logger.debug("Action: " + action); + final TransportResponseHandler delegate; + if (request.getParentTask().isSet()) { + // TODO: capture the connection instead so that we can cancel child tasks on the remote connections. + final Releasable unregisterChildNode = taskManager.registerChildNode( + request.getParentTask().getId(), + connection.getNode() + ); + delegate = new TransportResponseHandler() { + @Override + public void handleResponse(T response) { + unregisterChildNode.close(); + traceableTransportResponseHandler.handleResponse(response); + } - @Override - public void handleException(TransportException exp) { - unregisterChildNode.close(); - handler.handleException(exp); - } + @Override + public void handleException(TransportException exp) { + unregisterChildNode.close(); + traceableTransportResponseHandler.handleException(exp); + } - @Override - public String executor() { - return handler.executor(); - } + @Override + public String executor() { + return traceableTransportResponseHandler.executor(); + } - @Override - public T read(StreamInput in) throws IOException { - return handler.read(in); - } + @Override + public T read(StreamInput in) throws IOException { + return traceableTransportResponseHandler.read(in); + } - @Override - public String toString() { - return getClass().getName() + "/[" + action + "]:" + handler.toString(); - } - }; - } else { - delegate = handler; - } - asyncSender.sendRequest(connection, action, request, options, delegate); - } catch (final Exception ex) { - // the caller might not handle this so we invoke the handler - final TransportException te; - if (ex instanceof TransportException) { - te = (TransportException) ex; - } else { - te = new TransportException("failure to send", ex); + @Override + public String toString() { + return getClass().getName() + "/[" + action + "]:" + handler.toString(); + } + }; + } else { + delegate = traceableTransportResponseHandler; + } + asyncSender.sendRequest(connection, action, request, options, delegate); + } catch (final Exception ex) { + // the caller might not handle this so we invoke the handler + final TransportException te; + if (ex instanceof TransportException) { + te = (TransportException) ex; + } else { + te = new TransportException("failure to send", ex); + } + traceableTransportResponseHandler.handleException(te); } - handler.handleException(te); } } @@ -1017,6 +1030,7 @@ private void sendRequestInternal( threadPool.executor(executor).execute(new AbstractRunnable() { @Override public void onRejection(Exception e) { + contextToNotify.handler().handleRejection(e); // if we get rejected during node shutdown we don't wanna bubble it up logger.debug( () -> new ParameterizedMessage( @@ -1029,6 +1043,7 @@ public void onRejection(Exception e) { @Override public void onFailure(Exception e) { + contextToNotify.handler().handleRejection(e); logger.warn( () -> new ParameterizedMessage( "failed to notify response handler on exception, action: {}", diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 25f453fe024ff..6e064f943ca07 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2300,9 +2300,8 @@ public static void afterClass() throws Exception { INSTANCE.printTestMessage("cleaning up after"); INSTANCE.afterInternal(true); checkStaticState(true); - StrictCheckSpanProcessor.validateTracingStateOnShutdown(); } - + StrictCheckSpanProcessor.validateTracingStateOnShutdown(); } finally { SUITE_SEED = null; currentCluster = null; From 4a4a8fa6c04fe57ef5037fd0d49c1c416a1f3e24 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 15 Sep 2023 22:00:28 +0530 Subject: [PATCH 036/388] Always create empty translog on replica for remote store enabled index (#10012) Signed-off-by: Sachin Kale --- .../opensearch/remotestore/RemoteStoreIT.java | 70 +++++++++++++++++++ .../opensearch/index/shard/IndexShard.java | 44 ++++++++---- .../index/translog/TranslogHeader.java | 6 +- 3 files changed, 107 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index bd019693f01ff..3ccf563941f9c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; @@ -33,16 +34,20 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.index.shard.RemoteStoreRefreshListener.LAST_N_METADATA_FILES_TO_KEEP; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.comparesEqualTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.oneOf; @@ -346,4 +351,69 @@ private void clearClusterBufferIntervalSetting(String clusterManagerName) { .setTransientSettings(Settings.builder().putNull(CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey())) .get(); } + + public void testRestoreSnapshotToIndexWithSameNameDifferentUUID() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + List dataNodes = internalCluster().startDataOnlyNodes(2); + + Path absolutePath = randomRepoPath().toAbsolutePath(); + assertAcked( + clusterAdmin().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", absolutePath)) + ); + + logger.info("--> Create index and ingest 50 docs"); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); + indexBulk(INDEX_NAME, 50); + flushAndRefresh(INDEX_NAME); + + String originalIndexUUID = client().admin() + .indices() + .prepareGetSettings(INDEX_NAME) + .get() + .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); + assertNotNull(originalIndexUUID); + assertNotEquals(IndexMetadata.INDEX_UUID_NA_VALUE, originalIndexUUID); + + ensureGreen(); + + logger.info("--> take a snapshot"); + client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setIndices(INDEX_NAME).setWaitForCompletion(true).get(); + + logger.info("--> wipe all indices"); + cluster().wipeIndices(INDEX_NAME); + + logger.info("--> Create index with the same name, different UUID"); + assertAcked( + prepareCreate(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 1)) + ); + + ensureGreen(TimeValue.timeValueSeconds(30), INDEX_NAME); + + String newIndexUUID = client().admin() + .indices() + .prepareGetSettings(INDEX_NAME) + .get() + .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); + assertNotNull(newIndexUUID); + assertNotEquals(IndexMetadata.INDEX_UUID_NA_VALUE, newIndexUUID); + assertNotEquals(newIndexUUID, originalIndexUUID); + + logger.info("--> close index"); + client().admin().indices().prepareClose(INDEX_NAME).get(); + + logger.info("--> restore all indices from the snapshot"); + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .execute() + .actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + flushAndRefresh(INDEX_NAME); + + ensureGreen(INDEX_NAME); + assertBusy(() -> { + assertHitCount(client(dataNodes.get(0)).prepareSearch(INDEX_NAME).setSize(0).get(), 50); + assertHitCount(client(dataNodes.get(1)).prepareSearch(INDEX_NAME).setSize(0).get(), 50); + }); + } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 8ed75330f938e..34c5ed2112482 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -196,6 +196,7 @@ import java.io.IOException; import java.io.PrintStream; import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.NoSuchFileException; import java.util.ArrayList; @@ -2355,19 +2356,38 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier, b synchronized (engineMutex) { assert currentEngineReference.get() == null : "engine is running"; verifyNotClosed(); - if (indexSettings.isRemoteStoreEnabled() && syncFromRemote) { - syncSegmentsFromRemoteSegmentStore(false); - } - if (indexSettings.isRemoteTranslogStoreEnabled() && shardRouting.primary()) { + if (indexSettings.isRemoteStoreEnabled()) { + // Download missing segments from remote segment store. if (syncFromRemote) { - syncRemoteTranslogAndUpdateGlobalCheckpoint(); - } else { - // we will enter this block when we do not want to recover from remote translog. - // currently only during snapshot restore, we are coming into this block. - // here, as while initiliazing remote translog we cannot skip downloading translog files, - // so before that step, we are deleting the translog files present in remote store. - deleteTranslogFilesFromRemoteTranslog(); - + syncSegmentsFromRemoteSegmentStore(false); + } + if (shardRouting.primary()) { + if (syncFromRemote) { + syncRemoteTranslogAndUpdateGlobalCheckpoint(); + } else { + // we will enter this block when we do not want to recover from remote translog. + // currently only during snapshot restore, we are coming into this block. + // here, as while initiliazing remote translog we cannot skip downloading translog files, + // so before that step, we are deleting the translog files present in remote store. + deleteTranslogFilesFromRemoteTranslog(); + } + } else if (syncFromRemote) { + // For replicas, when we download segments from remote segment store, we need to make sure that local + // translog is having the same UUID that is referred by the segments. If they are different, engine open + // fails with TranslogCorruptedException. It is safe to create empty translog for remote store enabled + // indices as replica would only need to read translog in failover scenario and we always fetch data + // from remote translog at the time of failover. + final SegmentInfos lastCommittedSegmentInfos = store().readLastCommittedSegmentsInfo(); + final String translogUUID = lastCommittedSegmentInfos.userData.get(TRANSLOG_UUID_KEY); + final long checkpoint = Long.parseLong(lastCommittedSegmentInfos.userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); + Translog.createEmptyTranslog( + shardPath().resolveTranslog(), + shardId(), + checkpoint, + getPendingPrimaryTerm(), + translogUUID, + FileChannel::open + ); } } // we must create a new engine under mutex (see IndexShard#snapshotStoreMetadata). diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index 42bda11d75783..7b5be9505f27a 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -147,7 +147,11 @@ static TranslogHeader read(final String translogUUID, final Path path, final Fil if (actualUUID.bytesEquals(expectedUUID) == false) { throw new TranslogCorruptedException( path.toString(), - "expected shard UUID " + expectedUUID + " but got: " + actualUUID + " this translog file belongs to a different translog" + "expected shard UUID " + + translogUUID + + " but got: " + + translogHeader.translogUUID + + " this translog file belongs to a different translog" ); } return translogHeader; From d34b35272fa19bc53cb0f6843a92d2d1ee4e3b47 Mon Sep 17 00:00:00 2001 From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> Date: Fri, 15 Sep 2023 22:03:25 +0530 Subject: [PATCH 037/388] Fix broken backward compatibility from 2.7 for IndexSorted field indices (#10045) * Fix broken backward comparibility from 2.7 for IndexSorted field indices Signed-off-by: gashutos * Adding CHANGELOG Signed-off-by: gashutos * Update server/src/main/java/org/opensearch/index/IndexSettings.java Co-authored-by: Andriy Redko Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> * Removing unwanted logs Signed-off-by: gashutos * Removing unwanted logs Signed-off-by: gashutos * Adding index sort as part of mixed cluster to test this scenario Signed-off-by: gashutos * Removing optimization disable logic Signed-off-by: gashutos * Correcting some comments & version check to before( V_2_7_0) instead onOrBefire(V_2_6_1) since Signed-off-by: gashutos * Resolving spotless check error Signed-off-by: gashutos * Fixing broken UT - last minute checkin without copile Signed-off-by: gashutos * Improving code coverage to make codcov happy Signed-off-by: gashutos * Correcting typos and adding more tests Signed-off-by: gashutos * Removing unwanted imports Signed-off-by: gashutos --------- Signed-off-by: gashutos Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + .../org/opensearch/backwards/IndexingIT.java | 32 ++++++-- .../org/opensearch/index/IndexSettings.java | 17 +++++ .../org/opensearch/index/IndexSortConfig.java | 8 +- .../index/fielddata/IndexFieldData.java | 7 ++ .../fielddata/IndexNumericFieldData.java | 21 +++++- .../opensearch/index/IndexServiceTests.java | 74 +++++++++++++++++++ .../AbstractFieldDataImplTestCase.java | 22 ++++++ 8 files changed, 173 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f3c3abd4162..633f005a0883d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725)) +- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/9725)) ### Security diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java index 686fc78dcec8a..13c2daeec37af 100644 --- a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java @@ -67,13 +67,14 @@ public class IndexingIT extends OpenSearchRestTestCase { protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version")); + private static final String TEST_MAPPING = createTestMapping(); private int indexDocs(String index, final int idStart, final int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { final int id = idStart + i; Request request = new Request("PUT", index + "/_doc/" + id); - request.setJsonEntity("{\"test\": \"test_" + randomAlphaOfLength(2) + "\"}"); + request.setJsonEntity("{\"test\": \"test_" + randomAlphaOfLength(2) + "\", \"sortfield\": \""+ randomIntBetween(0, numDocs) + "\"}"); assertOK(client().performRequest(request)); } return numDocs; @@ -129,9 +130,10 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .putList("index.sort.field", "sortfield") .put("index.routing.allocation.include._name", bwcNames); final String index = "test-index"; - createIndex(index, settings.build()); + createIndex(index, settings.build(), TEST_MAPPING); ensureNoInitializingShards(); // wait for all other shard activity to finish int docCount = 200; @@ -178,9 +180,10 @@ public void testIndexingWithReplicaOnBwcNodes() throws Exception { .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .putList("index.sort.field", "sortfield") .put("index.routing.allocation.exclude._name", bwcNames); final String index = "test-index"; - createIndex(index, settings.build()); + createIndex(index, settings.build(), TEST_MAPPING); ensureNoInitializingShards(); // wait for all other shard activity to finish printClusterRouting(); @@ -214,11 +217,12 @@ public void testIndexVersionPropagation() throws Exception { Settings.Builder settings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2) + .putList("index.sort.field", "sortfield") .put("index.routing.allocation.include._name", bwcNames); final String index = "indexversionprop"; final int minUpdates = 5; final int maxUpdates = 10; - createIndex(index, settings.build()); + createIndex(index, settings.build(), TEST_MAPPING); try (RestClient newNodeClient = buildClient(restClientSettings(), nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) { @@ -300,10 +304,11 @@ public void testSeqNoCheckpoints() throws Exception { Settings.Builder settings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2) + .putList("index.sort.field", "sortfield") .put("index.routing.allocation.include._name", bwcNames); final String index = "test"; - createIndex(index, settings.build()); + createIndex(index, settings.build(), TEST_MAPPING); try (RestClient newNodeClient = buildClient(restClientSettings(), nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) { int numDocs = 0; @@ -382,10 +387,11 @@ public void testUpdateSnapshotStatus() throws Exception { Settings.Builder settings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(5, 10)) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1) + .putList("index.sort.field", "sortfield") .put("index.routing.allocation.include._name", bwcNames); final String index = "test-snapshot-index"; - createIndex(index, settings.build()); + createIndex(index, settings.build(), TEST_MAPPING); indexDocs(index, 0, between(50, 100)); ensureGreen(index); assertOK(client().performRequest(new Request("POST", index + "/_refresh"))); @@ -419,7 +425,8 @@ public void testSyncedFlushTransition() throws Exception { createIndex(index, Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) - .put("index.routing.allocation.include._name", newNodes).build()); + .putList("index.sort.field", "sortfield") + .put("index.routing.allocation.include._name", newNodes).build(), TEST_MAPPING); ensureGreen(index); indexDocs(index, randomIntBetween(0, 100), between(1, 100)); try (RestClient oldNodeClient = buildClient(restClientSettings(), @@ -664,4 +671,15 @@ public String toString() { '}'; } } + + private static String createTestMapping() { + return " \"properties\": {\n" + + " \"test\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"sortfield\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }"; + } } diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 03c71351294d5..1e4224c314f05 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -63,6 +63,7 @@ import java.util.function.Function; import java.util.function.UnaryOperator; +import static org.opensearch.Version.V_2_7_0; import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING; @@ -660,6 +661,7 @@ public final class IndexSettings { private volatile long retentionLeaseMillis; private volatile String defaultSearchPipeline; + private final boolean widenIndexSortType; /** * The maximum age of a retention lease before it is considered expired. @@ -857,6 +859,13 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mergeOnFlushEnabled = scopedSettings.get(INDEX_MERGE_ON_FLUSH_ENABLED); setMergeOnFlushPolicy(scopedSettings.get(INDEX_MERGE_ON_FLUSH_POLICY)); defaultSearchPipeline = scopedSettings.get(DEFAULT_SEARCH_PIPELINE); + /* There was unintentional breaking change got introduced with [OpenSearch-6424](https://github.com/opensearch-project/OpenSearch/pull/6424) (version 2.7). + * For indices created prior version (prior to 2.7) which has IndexSort type, they used to type cast the SortField.Type + * to higher bytes size like integer to long. This behavior was changed from OpenSearch 2.7 version not to + * up cast the SortField to gain some sort query optimizations. + * Now this sortField (IndexSort) is stored in SegmentInfo and we need to maintain backward compatibility for them. + */ + widenIndexSortType = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).before(V_2_7_0); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio); scopedSettings.addSettingsUpdateConsumer( @@ -1652,4 +1661,12 @@ public String getDefaultSearchPipeline() { public void setDefaultSearchPipeline(String defaultSearchPipeline) { this.defaultSearchPipeline = defaultSearchPipeline; } + + /** + * Returns true if we need to maintain backward compatibility for index sorted indices created prior to version 2.7 + * @return boolean + */ + public boolean shouldWidenIndexSortType() { + return this.widenIndexSortType; + } } diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java index f73f96df4f9ad..83192052564f3 100644 --- a/server/src/main/java/org/opensearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/opensearch/index/IndexSortConfig.java @@ -143,6 +143,7 @@ private static MultiValueMode parseMultiValueMode(String value) { // visible for tests final FieldSortSpec[] sortSpecs; + final boolean shouldWidenIndexSortType; public IndexSortConfig(IndexSettings indexSettings) { final Settings settings = indexSettings.getSettings(); @@ -182,6 +183,7 @@ public IndexSortConfig(IndexSettings indexSettings) { sortSpecs[i].missingValue = missingValues.get(i); } } + this.shouldWidenIndexSortType = indexSettings.shouldWidenIndexSortType(); } /** @@ -230,7 +232,11 @@ public Sort buildIndexSort( if (fieldData == null) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]"); } - sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse); + if (this.shouldWidenIndexSortType == true) { + sortFields[i] = fieldData.wideSortField(sortSpec.missingValue, mode, null, reverse); + } else { + sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse); + } validateIndexSortField(sortFields[i]); } return new Sort(sortFields); diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java index f9db28a2c56fe..81d4ce2dd8772 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java @@ -94,6 +94,13 @@ public interface IndexFieldData { */ SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse); + /** + * Returns the {@link SortField} to use for index sorting where we widen the sort field type to higher or equal bytes. + */ + default SortField wideSortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { + return sortField(missingValue, sortMode, nested, reverse); + } + /** * Build a sort implementation specialized for aggregations. */ diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java index ae8ffd8fe6b97..b4e90b8ab570a 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java @@ -151,6 +151,25 @@ public final SortField sortField(Object missingValue, MultiValueMode sortMode, N return sortField(getNumericType(), missingValue, sortMode, nested, reverse); } + @Override + public final SortField wideSortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { + // This is to support backward compatibility, the minimum number of bytes prior to OpenSearch 2.7 were 16 bytes, + // i.e all sort fields were upcasted to Long/Double with 16 bytes. + // Now from OpenSearch 2.7, the minimum number of bytes for sort field is 8 bytes, so if it comes as SortField INT, + // we need to up cast it to LONG to support backward compatibility info stored in segment info + if (getNumericType().sortFieldType == SortField.Type.INT) { + XFieldComparatorSource source = comparatorSource(NumericType.LONG, missingValue, sortMode, nested); + SortedNumericSelector.Type selectorType = sortMode == MultiValueMode.MAX + ? SortedNumericSelector.Type.MAX + : SortedNumericSelector.Type.MIN; + SortField sortField = new SortedNumericSortField(getFieldName(), SortField.Type.LONG, reverse, selectorType); + sortField.setMissingValue(source.missingObject(missingValue, reverse)); + return sortField; + } + // If already more than INT, up cast not needed. + return sortField(getNumericType(), missingValue, sortMode, nested, reverse); + } + /** * Builds a {@linkplain BucketedSort} for the {@code targetNumericType}, * casting the values if their native type doesn't match. @@ -224,7 +243,7 @@ private XFieldComparatorSource comparatorSource( source = new IntValuesComparatorSource(this, missingValue, sortMode, nested); } if (targetNumericType != getNumericType()) { - source.disableSkipping(); // disable skipping logic for caste of sort field + source.disableSkipping(); // disable skipping logic for cast of sort field } return source; } diff --git a/server/src/test/java/org/opensearch/index/IndexServiceTests.java b/server/src/test/java/org/opensearch/index/IndexServiceTests.java index 1b8e1abb1bf1b..db9f4bd305c79 100644 --- a/server/src/test/java/org/opensearch/index/IndexServiceTests.java +++ b/server/src/test/java/org/opensearch/index/IndexServiceTests.java @@ -33,7 +33,9 @@ package org.opensearch.index; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; +import org.opensearch.Version; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.compress.CompressedXContent; @@ -526,4 +528,76 @@ public void testUpdateRemoteTranslogBufferIntervalDynamically() { indexMetadata = client().admin().cluster().prepareState().execute().actionGet().getState().metadata().index("test"); assertEquals("20s", indexMetadata.getSettings().get(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey())); } + + public void testIndexSort() { + Settings settings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "0ms") // disable + .putList("index.sort.field", "sortfield") + .build(); + try { + // Integer index sort should be remained to int sort type + IndexService index = createIndex("test", settings, createTestMapping("integer")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.INT); + + // Long index sort should be remained to long sort type + index = createIndex("test", settings, createTestMapping("long")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.LONG); + + // Float index sort should be remained to float sort type + index = createIndex("test", settings, createTestMapping("float")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.FLOAT); + + // Double index sort should be remained to double sort type + index = createIndex("test", settings, createTestMapping("double")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.DOUBLE); + + // String index sort should be remained to string sort type + index = createIndex("test", settings, createTestMapping("string")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.STRING); + } catch (IllegalArgumentException ex) { + assertEquals("failed to parse value [0ms] for setting [index.translog.sync_interval], must be >= [100ms]", ex.getMessage()); + } + } + + public void testIndexSortBackwardCompatible() { + Settings settings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "0ms") // disable + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.V_2_6_1) + .putList("index.sort.field", "sortfield") + .build(); + try { + // Integer index sort should be converted to long sort type + IndexService index = createIndex("test", settings, createTestMapping("integer")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.LONG); + + // Long index sort should be remained to long sort type + index = createIndex("test", settings, createTestMapping("long")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.LONG); + + // Float index sort should be remained to float sort type + index = createIndex("test", settings, createTestMapping("float")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.FLOAT); + + // Double index sort should be remained to double sort type + index = createIndex("test", settings, createTestMapping("double")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.DOUBLE); + + // String index sort should be remained to string sort type + index = createIndex("test", settings, createTestMapping("string")); + assertTrue(index.getIndexSortSupplier().get().getSort()[0].getType() == SortField.Type.STRING); + } catch (IllegalArgumentException ex) { + assertEquals("failed to parse value [0ms] for setting [index.translog.sync_interval], must be >= [100ms]", ex.getMessage()); + } + } + + private static String createTestMapping(String type) { + return " \"properties\": {\n" + + " \"test\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"sortfield\": {\n" + + " \"type\": \" + type + \"\n" + + " }\n" + + " }"; + } } diff --git a/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataImplTestCase.java b/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataImplTestCase.java index 1ffacf98a6836..2b44e759f4ff9 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataImplTestCase.java +++ b/server/src/test/java/org/opensearch/index/fielddata/AbstractFieldDataImplTestCase.java @@ -39,6 +39,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.util.BytesRef; import org.opensearch.core.common.Strings; @@ -144,6 +145,27 @@ public void testSingleValueAllSet() throws Exception { } } + public void testWideSortField() throws Exception { + if (this instanceof NoOrdinalsStringFieldDataTests || this instanceof PagedBytesStringFieldDataTests) { + return; // Numeric types are not supported there. + } + // integer to long widening should happen + IndexFieldData indexFieldData = getForField("int", "value"); + SortField sortField = indexFieldData.wideSortField(null, MultiValueMode.MIN, null, false); + assertTrue(((SortedNumericSortField) sortField).getNumericType() == SortField.Type.LONG); + + // long to long no widening should happen + indexFieldData = getForField("long", "value"); + sortField = indexFieldData.wideSortField(null, MultiValueMode.MIN, null, false); + assertTrue(((SortedNumericSortField) sortField).getNumericType() == SortField.Type.LONG); + + // float to float no widening should happen + indexFieldData = getForField("float", "value"); + sortField = indexFieldData.wideSortField(null, MultiValueMode.MIN, null, false); + assertTrue(((SortedNumericSortField) sortField).getNumericType() == SortField.Type.FLOAT); + + } + protected abstract void fillSingleValueWithMissing() throws Exception; public void assertValues(SortedBinaryDocValues values, int docId, BytesRef... actualValues) throws IOException { From 70a582fcbe82e9b4250e29cbf7d6db470370906a Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 15 Sep 2023 15:38:42 -0700 Subject: [PATCH 038/388] Add async read support for S3 plugin (#9694) * Add async read support for S3 plugin Signed-off-by: Kunal Kotwani (cherry picked from commit 03ddc8a6b07221b1ed60ad5627939d1f957b1c49) Signed-off-by: Kunal Kotwani * Move functionality to S3BlobContainer Signed-off-by: Kunal Kotwani --------- Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 1 + .../repositories/s3/S3BlobContainer.java | 109 +++++- .../repositories/s3/utils/HttpRangeUtils.java | 22 ++ .../s3/S3BlobStoreContainerTests.java | 310 +++++++++++++++++- .../s3/utils/HttpRangeUtilsTests.java | 29 ++ 5 files changed, 458 insertions(+), 13 deletions(-) create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/utils/HttpRangeUtilsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 633f005a0883d..337573480adfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added - Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681)) +- Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index bb1643faecc95..2911a018df337 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -32,6 +32,8 @@ package org.opensearch.repositories.s3; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -44,10 +46,15 @@ import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectAttributesRequest; +import software.amazon.awssdk.services.s3.model.GetObjectAttributesResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.ObjectAttributes; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Error; @@ -63,6 +70,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; import org.opensearch.common.StreamContext; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -75,11 +83,13 @@ import org.opensearch.common.blobstore.support.AbstractBlobContainer; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.io.InputStreamContainer; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.repositories.s3.async.UploadRequest; +import org.opensearch.repositories.s3.utils.HttpRangeUtils; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -212,9 +222,45 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener comp } } + @ExperimentalApi @Override public void readBlobAsync(String blobName, ActionListener listener) { - throw new UnsupportedOperationException(); + try (AmazonAsyncS3Reference amazonS3Reference = SocketAccess.doPrivileged(blobStore::asyncClientReference)) { + final S3AsyncClient s3AsyncClient = amazonS3Reference.get().client(); + final String bucketName = blobStore.bucket(); + + final GetObjectAttributesResponse blobMetadata = getBlobMetadata(s3AsyncClient, bucketName, blobName).get(); + + final long blobSize = blobMetadata.objectSize(); + final int numberOfParts = blobMetadata.objectParts().totalPartsCount(); + final String blobChecksum = blobMetadata.checksum().checksumCRC32(); + + final List blobPartStreams = new ArrayList<>(); + final List> blobPartInputStreamFutures = new ArrayList<>(); + // S3 multipart files use 1 to n indexing + for (int partNumber = 1; partNumber <= numberOfParts; partNumber++) { + blobPartInputStreamFutures.add(getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobName, partNumber)); + } + + CompletableFuture.allOf(blobPartInputStreamFutures.toArray(CompletableFuture[]::new)).whenComplete((unused, throwable) -> { + if (throwable == null) { + listener.onResponse( + new ReadContext( + blobSize, + blobPartInputStreamFutures.stream().map(CompletableFuture::join).collect(Collectors.toList()), + blobChecksum + ) + ); + } else { + Exception ex = throwable.getCause() instanceof Exception + ? (Exception) throwable.getCause() + : new Exception(throwable.getCause()); + listener.onFailure(ex); + } + }); + } catch (Exception ex) { + listener.onFailure(SdkException.create("Error occurred while fetching blob parts from the repository", ex)); + } } public boolean remoteIntegrityCheckSupported() { @@ -633,4 +679,65 @@ static Tuple numberOfMultiparts(final long totalSize, final long par return Tuple.tuple(parts + 1, remaining); } } + + /** + * Fetches a part of the blob from the S3 bucket and transforms it to an {@link InputStreamContainer}, which holds + * the stream and its related metadata. + * @param s3AsyncClient Async client to be utilized to fetch the object part + * @param bucketName Name of the S3 bucket + * @param blobName Identifier of the blob for which the parts will be fetched + * @param partNumber Part number for the blob to be retrieved + * @return A future of {@link InputStreamContainer} containing the stream and stream metadata. + */ + CompletableFuture getBlobPartInputStreamContainer( + S3AsyncClient s3AsyncClient, + String bucketName, + String blobName, + int partNumber + ) { + final GetObjectRequest.Builder getObjectRequestBuilder = GetObjectRequest.builder() + .bucket(bucketName) + .key(blobName) + .partNumber(partNumber); + + return SocketAccess.doPrivileged( + () -> s3AsyncClient.getObject(getObjectRequestBuilder.build(), AsyncResponseTransformer.toBlockingInputStream()) + .thenApply(S3BlobContainer::transformResponseToInputStreamContainer) + ); + } + + /** + * Transforms the stream response object from S3 into an {@link InputStreamContainer} + * @param streamResponse Response stream object from S3 + * @return {@link InputStreamContainer} containing the stream and stream metadata + */ + // Package-Private for testing. + static InputStreamContainer transformResponseToInputStreamContainer(ResponseInputStream streamResponse) { + final GetObjectResponse getObjectResponse = streamResponse.response(); + final String contentRange = getObjectResponse.contentRange(); + final Long contentLength = getObjectResponse.contentLength(); + if (contentRange == null || contentLength == null) { + throw SdkException.builder().message("Failed to fetch required metadata for blob part").build(); + } + final Long offset = HttpRangeUtils.getStartOffsetFromRangeHeader(getObjectResponse.contentRange()); + return new InputStreamContainer(streamResponse, getObjectResponse.contentLength(), offset); + } + + /** + * Retrieves the metadata like checksum, object size and parts for the provided blob within the S3 bucket. + * @param s3AsyncClient Async client to be utilized to fetch the metadata + * @param bucketName Name of the S3 bucket + * @param blobName Identifier of the blob for which the metadata will be fetched + * @return A future containing the metadata within {@link GetObjectAttributesResponse} + */ + CompletableFuture getBlobMetadata(S3AsyncClient s3AsyncClient, String bucketName, String blobName) { + // Fetch blob metadata - part info, size, checksum + final GetObjectAttributesRequest getObjectAttributesRequest = GetObjectAttributesRequest.builder() + .bucket(bucketName) + .key(blobName) + .objectAttributes(ObjectAttributes.CHECKSUM, ObjectAttributes.OBJECT_SIZE, ObjectAttributes.OBJECT_PARTS) + .build(); + + return SocketAccess.doPrivileged(() -> s3AsyncClient.getObjectAttributes(getObjectAttributesRequest)); + } } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/utils/HttpRangeUtils.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/utils/HttpRangeUtils.java index 40aec7d52847b..2e2fc9b86a45b 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/utils/HttpRangeUtils.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/utils/HttpRangeUtils.java @@ -8,7 +8,29 @@ package org.opensearch.repositories.s3.utils; +import software.amazon.awssdk.core.exception.SdkException; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public final class HttpRangeUtils { + private static final Pattern RANGE_PATTERN = Pattern.compile("^bytes\\s+(\\d+)-\\d+[/\\d*]+$"); + + /** + * Parses the content range header string value to calculate the start (offset) of the HTTP response. + * Tests against the RFC9110 specification of content range string. + * Sample values: "bytes 0-10/200", "bytes 0-10/*" + * Details here + * @param headerValue Header content range string value from the HTTP response + * @return Start (Offset) value of the HTTP response + */ + public static Long getStartOffsetFromRangeHeader(String headerValue) { + Matcher matcher = RANGE_PATTERN.matcher(headerValue); + if (!matcher.find()) { + throw SdkException.create("Regex match for Content-Range header {" + headerValue + "} failed", new RuntimeException()); + } + return Long.parseLong(matcher.group(1)); + } /** * Provides a byte range string per RFC 9110 diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index 1c4936cae7eba..a87c060dcc60a 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -32,11 +32,15 @@ package org.opensearch.repositories.s3; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.Checksum; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedPart; @@ -44,6 +48,11 @@ import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectAttributesParts; +import software.amazon.awssdk.services.s3.model.GetObjectAttributesRequest; +import software.amazon.awssdk.services.s3.model.GetObjectAttributesResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; @@ -61,15 +70,18 @@ import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; -import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.action.LatchedActionListener; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.blobstore.DeleteResult; +import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.io.InputStreamContainer; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.repositories.s3.async.AsyncTransferManager; import org.opensearch.test.OpenSearchTestCase; import java.io.ByteArrayInputStream; @@ -86,14 +98,19 @@ import java.util.NoSuchElementException; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -882,17 +899,6 @@ public void onFailure(Exception e) {} } } - public void testAsyncBlobDownload() { - final S3BlobStore blobStore = mock(S3BlobStore.class); - final BlobPath blobPath = mock(BlobPath.class); - final String blobName = "test-blob"; - - final UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> { - final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); - blobContainer.readBlobAsync(blobName, new PlainActionFuture<>()); - }); - } - public void testListBlobsByPrefixInLexicographicOrderWithNegativeLimit() throws IOException { testListBlobsByPrefixInLexicographicOrder(-5, 0, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); } @@ -912,4 +918,284 @@ public void testListBlobsByPrefixInLexicographicOrderWithLimitGreaterThanPageSiz public void testListBlobsByPrefixInLexicographicOrderWithLimitGreaterThanNumberOfRecords() throws IOException { testListBlobsByPrefixInLexicographicOrder(12, 2, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); } + + public void testReadBlobAsync() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final String blobName = randomAlphaOfLengthBetween(1, 10); + final String checksum = randomAlphaOfLength(10); + + final long objectSize = 100L; + final int objectPartCount = 10; + final int partSize = 10; + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference( + AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null) + ); + final AsyncTransferManager asyncTransferManager = new AsyncTransferManager( + 10000L, + mock(ExecutorService.class), + mock(ExecutorService.class) + ); + final S3BlobStore blobStore = mock(S3BlobStore.class); + final BlobPath blobPath = new BlobPath(); + + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.serverSideEncryption()).thenReturn(false); + when(blobStore.asyncClientReference()).thenReturn(amazonAsyncS3Reference); + when(blobStore.getAsyncTransferManager()).thenReturn(asyncTransferManager); + + CompletableFuture getObjectAttributesResponseCompletableFuture = new CompletableFuture<>(); + getObjectAttributesResponseCompletableFuture.complete( + GetObjectAttributesResponse.builder() + .checksum(Checksum.builder().checksumCRC32(checksum).build()) + .objectSize(objectSize) + .objectParts(GetObjectAttributesParts.builder().totalPartsCount(objectPartCount).build()) + .build() + ); + when(s3AsyncClient.getObjectAttributes(any(GetObjectAttributesRequest.class))).thenReturn( + getObjectAttributesResponseCompletableFuture + ); + + mockObjectPartResponse(s3AsyncClient, bucketName, blobName, objectPartCount, partSize, objectSize); + + CountDownLatch countDownLatch = new CountDownLatch(1); + CountingCompletionListener readContextActionListener = new CountingCompletionListener<>(); + LatchedActionListener listener = new LatchedActionListener<>(readContextActionListener, countDownLatch); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + blobContainer.readBlobAsync(blobName, listener); + countDownLatch.await(); + + assertEquals(1, readContextActionListener.getResponseCount()); + assertEquals(0, readContextActionListener.getFailureCount()); + ReadContext readContext = readContextActionListener.getResponse(); + assertEquals(objectPartCount, readContext.getNumberOfParts()); + assertEquals(checksum, readContext.getBlobChecksum()); + assertEquals(objectSize, readContext.getBlobSize()); + + for (int partNumber = 1; partNumber < objectPartCount; partNumber++) { + InputStreamContainer inputStreamContainer = readContext.getPartStreams().get(partNumber); + final int offset = partNumber * partSize; + assertEquals(partSize, inputStreamContainer.getContentLength()); + assertEquals(offset, inputStreamContainer.getOffset()); + assertEquals(partSize, inputStreamContainer.getInputStream().readAllBytes().length); + } + } + + public void testReadBlobAsyncFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final String blobName = randomAlphaOfLengthBetween(1, 10); + final String checksum = randomAlphaOfLength(10); + + final long objectSize = 100L; + final int objectPartCount = 10; + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference( + AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null) + ); + final AsyncTransferManager asyncTransferManager = new AsyncTransferManager( + 10000L, + mock(ExecutorService.class), + mock(ExecutorService.class) + ); + final S3BlobStore blobStore = mock(S3BlobStore.class); + final BlobPath blobPath = new BlobPath(); + + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.serverSideEncryption()).thenReturn(false); + when(blobStore.asyncClientReference()).thenReturn(amazonAsyncS3Reference); + when(blobStore.getAsyncTransferManager()).thenReturn(asyncTransferManager); + + CompletableFuture getObjectAttributesResponseCompletableFuture = new CompletableFuture<>(); + getObjectAttributesResponseCompletableFuture.complete( + GetObjectAttributesResponse.builder() + .checksum(Checksum.builder().checksumCRC32(checksum).build()) + .objectSize(objectSize) + .objectParts(GetObjectAttributesParts.builder().totalPartsCount(objectPartCount).build()) + .build() + ); + when(s3AsyncClient.getObjectAttributes(any(GetObjectAttributesRequest.class))).thenThrow(new RuntimeException()); + + CountDownLatch countDownLatch = new CountDownLatch(1); + CountingCompletionListener readContextActionListener = new CountingCompletionListener<>(); + LatchedActionListener listener = new LatchedActionListener<>(readContextActionListener, countDownLatch); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + blobContainer.readBlobAsync(blobName, listener); + countDownLatch.await(); + + assertEquals(0, readContextActionListener.getResponseCount()); + assertEquals(1, readContextActionListener.getFailureCount()); + } + + public void testGetBlobMetadata() throws Exception { + final String checksum = randomAlphaOfLengthBetween(1, 10); + final long objectSize = 100L; + final int objectPartCount = 10; + final String blobName = randomAlphaOfLengthBetween(1, 10); + final String bucketName = randomAlphaOfLengthBetween(1, 10); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final S3BlobStore blobStore = mock(S3BlobStore.class); + final BlobPath blobPath = new BlobPath(); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.serverSideEncryption()).thenReturn(false); + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CompletableFuture getObjectAttributesResponseCompletableFuture = new CompletableFuture<>(); + getObjectAttributesResponseCompletableFuture.complete( + GetObjectAttributesResponse.builder() + .checksum(Checksum.builder().checksumCRC32(checksum).build()) + .objectSize(objectSize) + .objectParts(GetObjectAttributesParts.builder().totalPartsCount(objectPartCount).build()) + .build() + ); + when(s3AsyncClient.getObjectAttributes(any(GetObjectAttributesRequest.class))).thenReturn( + getObjectAttributesResponseCompletableFuture + ); + + CompletableFuture responseFuture = blobContainer.getBlobMetadata(s3AsyncClient, bucketName, blobName); + GetObjectAttributesResponse objectAttributesResponse = responseFuture.get(); + + assertEquals(checksum, objectAttributesResponse.checksum().checksumCRC32()); + assertEquals(Long.valueOf(objectSize), objectAttributesResponse.objectSize()); + assertEquals(Integer.valueOf(objectPartCount), objectAttributesResponse.objectParts().totalPartsCount()); + } + + public void testGetBlobPartInputStream() throws Exception { + final String blobName = randomAlphaOfLengthBetween(1, 10); + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final long contentLength = 10L; + final String contentRange = "bytes 0-10/100"; + final InputStream inputStream = ResponseInputStream.nullInputStream(); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final S3BlobStore blobStore = mock(S3BlobStore.class); + final BlobPath blobPath = new BlobPath(); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.serverSideEncryption()).thenReturn(false); + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + GetObjectResponse getObjectResponse = GetObjectResponse.builder().contentLength(contentLength).contentRange(contentRange).build(); + + CompletableFuture> getObjectPartResponse = new CompletableFuture<>(); + ResponseInputStream responseInputStream = new ResponseInputStream<>(getObjectResponse, inputStream); + getObjectPartResponse.complete(responseInputStream); + + when( + s3AsyncClient.getObject( + any(GetObjectRequest.class), + ArgumentMatchers.>>any() + ) + ).thenReturn(getObjectPartResponse); + + InputStreamContainer inputStreamContainer = blobContainer.getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobName, 0) + .get(); + + assertEquals(0, inputStreamContainer.getOffset()); + assertEquals(contentLength, inputStreamContainer.getContentLength()); + assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); + } + + public void testTransformResponseToInputStreamContainer() throws Exception { + final String contentRange = "bytes 0-10/100"; + final long contentLength = 10L; + final InputStream inputStream = ResponseInputStream.nullInputStream(); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + + GetObjectResponse getObjectResponse = GetObjectResponse.builder().contentLength(contentLength).build(); + + ResponseInputStream responseInputStreamNoRange = new ResponseInputStream<>(getObjectResponse, inputStream); + assertThrows(SdkException.class, () -> S3BlobContainer.transformResponseToInputStreamContainer(responseInputStreamNoRange)); + + getObjectResponse = GetObjectResponse.builder().contentRange(contentRange).build(); + ResponseInputStream responseInputStreamNoContentLength = new ResponseInputStream<>( + getObjectResponse, + inputStream + ); + assertThrows(SdkException.class, () -> S3BlobContainer.transformResponseToInputStreamContainer(responseInputStreamNoContentLength)); + + getObjectResponse = GetObjectResponse.builder().contentRange(contentRange).contentLength(contentLength).build(); + ResponseInputStream responseInputStream = new ResponseInputStream<>(getObjectResponse, inputStream); + InputStreamContainer inputStreamContainer = S3BlobContainer.transformResponseToInputStreamContainer(responseInputStream); + assertEquals(contentLength, inputStreamContainer.getContentLength()); + assertEquals(0, inputStreamContainer.getOffset()); + assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); + } + + private void mockObjectPartResponse( + S3AsyncClient s3AsyncClient, + String bucketName, + String blobName, + int totalNumberOfParts, + int partSize, + long objectSize + ) { + for (int partNumber = 1; partNumber <= totalNumberOfParts; partNumber++) { + final int start = (partNumber - 1) * partSize; + final int end = partNumber * partSize; + final String contentRange = "bytes " + start + "-" + end + "/" + objectSize; + final InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(partSize)); + + GetObjectResponse getObjectResponse = GetObjectResponse.builder() + .contentLength((long) partSize) + .contentRange(contentRange) + .build(); + + CompletableFuture> getObjectPartResponse = new CompletableFuture<>(); + ResponseInputStream responseInputStream = new ResponseInputStream<>(getObjectResponse, inputStream); + getObjectPartResponse.complete(responseInputStream); + + GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(blobName).partNumber(partNumber).build(); + + when( + s3AsyncClient.getObject( + eq(getObjectRequest), + ArgumentMatchers.>>any() + ) + ).thenReturn(getObjectPartResponse); + } + } + + private static class CountingCompletionListener implements ActionListener { + private int responseCount; + private int failureCount; + private T response; + private Exception exception; + + @Override + public void onResponse(T response) { + this.response = response; + responseCount++; + } + + @Override + public void onFailure(Exception e) { + exception = e; + failureCount++; + } + + public int getResponseCount() { + return responseCount; + } + + public int getFailureCount() { + return failureCount; + } + + public T getResponse() { + return response; + } + + public Exception getException() { + return exception; + } + } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/utils/HttpRangeUtilsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/utils/HttpRangeUtilsTests.java new file mode 100644 index 0000000000000..9a4267c5266e5 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/utils/HttpRangeUtilsTests.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.utils; + +import software.amazon.awssdk.core.exception.SdkException; + +import org.opensearch.test.OpenSearchTestCase; + +public final class HttpRangeUtilsTests extends OpenSearchTestCase { + + public void testFromHttpRangeHeader() { + String headerValue = "bytes 0-10/200"; + Long offset = HttpRangeUtils.getStartOffsetFromRangeHeader(headerValue); + assertEquals(0L, offset.longValue()); + + headerValue = "bytes 0-10/*"; + offset = HttpRangeUtils.getStartOffsetFromRangeHeader(headerValue); + assertEquals(0L, offset.longValue()); + + final String invalidHeaderValue = "bytes */*"; + assertThrows(SdkException.class, () -> HttpRangeUtils.getStartOffsetFromRangeHeader(invalidHeaderValue)); + } +} From 5fdd418be5717cc91023b071075fb1ab525ba8e4 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Sat, 16 Sep 2023 12:59:56 +0530 Subject: [PATCH 039/388] skip overriding routing table when it already contains entries with remote recovery source (#9962) * skip overriding routing table when it already contains entries with remote recovery source Signed-off-by: bansvaru --- .../gateway/ClusterStateUpdaters.java | 17 +- .../recovery/RemoteStoreRestoreService.java | 2 +- .../gateway/ClusterStateUpdatersTests.java | 320 ++++++++++++++++++ 3 files changed, 337 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java index 1563ac84bdd1c..4c562b348f141 100644 --- a/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java +++ b/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java @@ -41,6 +41,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.settings.ClusterSettings; @@ -120,7 +121,21 @@ static ClusterState updateRoutingTable(final ClusterState state) { // initialize all index routing tables as empty final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(state.routingTable()); for (final IndexMetadata cursor : state.metadata().indices().values()) { - routingTableBuilder.addAsRecovery(cursor); + // Whether IndexMetadata is recovered from local disk or remote it doesn't matter to us at this point. + // We are only concerned about index data recovery here. Which is why we only check for remote store enabled and not for remote + // cluster state enabled. + if (cursor.getSettings().getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) == false + || state.routingTable().hasIndex(cursor.getIndex()) == false + || state.routingTable() + .index(cursor.getIndex()) + .shardsMatchingPredicateCount( + shardRouting -> shardRouting.primary() + // We need to ensure atleast one of the primaries is being recovered from remote. + // This ensures we have gone through the RemoteStoreRestoreService and routing table is updated + && shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) == 0) { + routingTableBuilder.addAsRecovery(cursor); + } } // start with 0 based versions for routing table routingTableBuilder.version(0); diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java index d05242a3aeaf7..94fd08b99ac58 100644 --- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java +++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java @@ -187,7 +187,7 @@ private RemoteRestoreResult executeRestore( IndexMetadata indexMetadata = indexMetadataEntry.getValue().v2(); boolean metadataFromRemoteStore = indexMetadataEntry.getValue().v1(); IndexMetadata updatedIndexMetadata = indexMetadata; - if (restoreAllShards || metadataFromRemoteStore) { + if (metadataFromRemoteStore == false && restoreAllShards) { updatedIndexMetadata = IndexMetadata.builder(indexMetadata) .state(IndexMetadata.State.OPEN) .version(1 + indexMetadata.getVersion()) diff --git a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java index c83da46b23fb1..9b3fd45245ef7 100644 --- a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java @@ -40,6 +40,10 @@ import org.opensearch.cluster.metadata.MetadataIndexStateService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RecoverySource; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.ClusterSettings; @@ -48,10 +52,12 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.set.Sets; import org.opensearch.core.index.Index; +import org.opensearch.repositories.IndexId; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Function; @@ -269,6 +275,320 @@ public void testUpdateRoutingTable() { } } + public void testSkipRoutingTableUpdateWhenRemoteRecovery() { + final int numOfShards = randomIntBetween(1, 10); + + final IndexMetadata remoteMetadata = createIndexMetadata( + "test-remote", + Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .build() + ); + + // Test remote index routing table is generated with ExistingStoreRecoverySource if no routing table is present + { + final Index index = remoteMetadata.getIndex(); + final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(remoteMetadata, false).build()) + .build(); + final ClusterState newState = updateRoutingTable(initialState); + IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex()); + assertTrue(newState.routingTable().hasIndex(index)); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.INDEX_CREATED) + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED) + ) + ); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource + ) + ); + + } + + // Test remote index routing table is overridden if recovery source is not RemoteStoreRecoverySource + { + IndexRoutingTable.Builder remoteBuilderWithoutRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex()) + .initializeAsNew(remoteMetadata); + final Index index = remoteMetadata.getIndex(); + final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(remoteMetadata, false).build()) + .routingTable(new RoutingTable.Builder().add(remoteBuilderWithoutRemoteRecovery.build()).build()) + .build(); + assertTrue(initialState.routingTable().hasIndex(index)); + final ClusterState newState = updateRoutingTable(initialState); + IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex()); + assertTrue(newState.routingTable().hasIndex(index)); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.INDEX_CREATED) + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED) + ) + ); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource + ) + ); + + } + + // Test routing table update is skipped for a remote index + { + IndexRoutingTable.Builder remoteBuilderWithRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex()) + .initializeAsRemoteStoreRestore( + remoteMetadata, + new RecoverySource.RemoteStoreRecoverySource( + UUIDs.randomBase64UUID(), + remoteMetadata.getCreationVersion(), + new IndexId(remoteMetadata.getIndex().getName(), remoteMetadata.getIndexUUID()) + ), + new HashMap<>(), + true + ); + final Index index = remoteMetadata.getIndex(); + final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(remoteMetadata, false).build()) + .routingTable(new RoutingTable.Builder().add(remoteBuilderWithRemoteRecovery.build()).build()) + .build(); + assertTrue(initialState.routingTable().hasIndex(index)); + final ClusterState newState = updateRoutingTable(initialState); + IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex()); + assertTrue(newState.routingTable().hasIndex(index)); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED) + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED) + ) + ); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) + ); + + } + + // Test reset routing table for 2 indices - one remote and one non remote. + // Routing table for non remote index should be updated and remote index routing table should remain intact + { + final IndexMetadata nonRemoteMetadata = createIndexMetadata( + "test-nonremote", + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards).build() + ); + IndexRoutingTable.Builder remoteBuilderWithRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex()) + .initializeAsRemoteStoreRestore( + remoteMetadata, + new RecoverySource.RemoteStoreRecoverySource( + UUIDs.randomBase64UUID(), + remoteMetadata.getCreationVersion(), + new IndexId(remoteMetadata.getIndex().getName(), remoteMetadata.getIndexUUID()) + ), + new HashMap<>(), + true + ); + IndexRoutingTable.Builder nonRemoteBuilderWithoutRemoteRecovery = new IndexRoutingTable.Builder(nonRemoteMetadata.getIndex()) + .initializeAsNew(nonRemoteMetadata); + final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(remoteMetadata, false).build()) + .metadata(Metadata.builder().put(nonRemoteMetadata, false).build()) + .routingTable( + new RoutingTable.Builder().add(remoteBuilderWithRemoteRecovery.build()) + .add(nonRemoteBuilderWithoutRemoteRecovery.build()) + .build() + ) + .build(); + assertTrue(initialState.routingTable().hasIndex(remoteMetadata.getIndex())); + assertTrue(initialState.routingTable().hasIndex(nonRemoteMetadata.getIndex())); + final ClusterState newState = updateRoutingTable(initialState); + assertTrue(newState.routingTable().hasIndex(remoteMetadata.getIndex())); + assertTrue(newState.routingTable().hasIndex(nonRemoteMetadata.getIndex())); + IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex()); + IndexRoutingTable newNonRemoteIndexRoutingTable = newState.routingTable().index(nonRemoteMetadata.getIndex()); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED) + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED) + ) + ); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) + ); + assertEquals( + 0, + newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.INDEX_CREATED) + ) + ); + assertEquals( + numOfShards, + newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED) + ) + ); + assertEquals( + 0, + newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) + ); + assertEquals( + numOfShards, + newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource + ) + ); + } + + // Test reset routing table for 2 indices, both remote backed but only once index has RemoteStoreRecoverySource. + // Routing table for only remote index without RemoteStoreRecoverySource should be updated + { + final IndexMetadata remoteWithoutRemoteRecoveryMetadata = createIndexMetadata( + "test-remote-without-recovery", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) + .build() + ); + IndexRoutingTable.Builder remoteBuilderWithRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex()) + .initializeAsRemoteStoreRestore( + remoteMetadata, + new RecoverySource.RemoteStoreRecoverySource( + UUIDs.randomBase64UUID(), + remoteMetadata.getCreationVersion(), + new IndexId(remoteMetadata.getIndex().getName(), remoteMetadata.getIndexUUID()) + ), + new HashMap<>(), + true + ); + IndexRoutingTable.Builder remoteBuilderWithoutRemoteRecovery = new IndexRoutingTable.Builder( + remoteWithoutRemoteRecoveryMetadata.getIndex() + ).initializeAsNew(remoteWithoutRemoteRecoveryMetadata); + final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(remoteMetadata, false).build()) + .metadata(Metadata.builder().put(remoteWithoutRemoteRecoveryMetadata, false).build()) + .routingTable( + new RoutingTable.Builder().add(remoteBuilderWithRemoteRecovery.build()) + .add(remoteBuilderWithoutRemoteRecovery.build()) + .build() + ) + .build(); + assertTrue(initialState.routingTable().hasIndex(remoteMetadata.getIndex())); + assertTrue(initialState.routingTable().hasIndex(remoteWithoutRemoteRecoveryMetadata.getIndex())); + final ClusterState newState = updateRoutingTable(initialState); + assertTrue(newState.routingTable().hasIndex(remoteMetadata.getIndex())); + assertTrue(newState.routingTable().hasIndex(remoteWithoutRemoteRecoveryMetadata.getIndex())); + IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex()); + IndexRoutingTable newRemoteWithoutRemoteRecoveryIndexRoutingTable = newState.routingTable() + .index(remoteWithoutRemoteRecoveryMetadata.getIndex()); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED) + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED) + ) + ); + assertEquals( + 0, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource + ) + ); + assertEquals( + numOfShards, + newRemoteIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) + ); + assertEquals( + 0, + newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.INDEX_CREATED) + ) + ); + assertEquals( + numOfShards, + newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED) + ) + ); + assertEquals( + 0, + newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource + ) + ); + assertEquals( + numOfShards, + newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount( + shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource + ) + ); + } + } + public void testMixCurrentAndRecoveredState() { final ClusterState currentState = ClusterState.builder(ClusterState.EMPTY_STATE) .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK).build()) From d7aa6dd9c7759bbfedd7263d453d3ac6c92fab7d Mon Sep 17 00:00:00 2001 From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> Date: Sat, 16 Sep 2023 20:49:23 +0530 Subject: [PATCH 040/388] Finxing broken UT (#10087) Signed-off-by: Gohel Co-authored-by: Gohel --- server/src/main/java/org/opensearch/index/IndexService.java | 2 ++ .../src/main/java/org/opensearch/index/IndexSortConfig.java | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 80ead0a333ba3..38456ad3e43a5 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -240,8 +240,10 @@ public IndexService( if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. // The sort order is validated right after the merge of the mapping later in the process. + boolean shouldWidenIndexSortType = this.indexSettings.shouldWidenIndexSortType(); this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() .buildIndexSort( + shouldWidenIndexSortType, mapperService::fieldType, (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup) ); diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java index 83192052564f3..763ed34a1e6a6 100644 --- a/server/src/main/java/org/opensearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/opensearch/index/IndexSortConfig.java @@ -143,7 +143,6 @@ private static MultiValueMode parseMultiValueMode(String value) { // visible for tests final FieldSortSpec[] sortSpecs; - final boolean shouldWidenIndexSortType; public IndexSortConfig(IndexSettings indexSettings) { final Settings settings = indexSettings.getSettings(); @@ -183,7 +182,6 @@ public IndexSortConfig(IndexSettings indexSettings) { sortSpecs[i].missingValue = missingValues.get(i); } } - this.shouldWidenIndexSortType = indexSettings.shouldWidenIndexSortType(); } /** @@ -202,6 +200,7 @@ public boolean hasPrimarySortOnField(String field) { * or returns null if this index has no sort. */ public Sort buildIndexSort( + boolean shouldWidenIndexSortType, Function fieldTypeLookup, BiFunction, IndexFieldData> fieldDataLookup ) { @@ -232,7 +231,7 @@ public Sort buildIndexSort( if (fieldData == null) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]"); } - if (this.shouldWidenIndexSortType == true) { + if (shouldWidenIndexSortType == true) { sortFields[i] = fieldData.wideSortField(sortSpec.missingValue, mode, null, reverse); } else { sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse); From b5cc00222816429ed673d9fb46c434cae1be862b Mon Sep 17 00:00:00 2001 From: Ashish Date: Mon, 18 Sep 2023 21:37:41 +0530 Subject: [PATCH 041/388] Optimize read write lock constructs during translog upload to remote store (#9636) Signed-off-by: Ashish Singh --- .../index/translog/RemoteFsTranslog.java | 85 +++++++++++++------ .../TranslogCheckpointTransferSnapshot.java | 1 - .../transfer/TranslogTransferManager.java | 2 +- .../listener/TranslogTransferListener.java | 2 +- .../index/translog/RemoteFsTranslogTests.java | 1 - .../TranslogTransferManagerTests.java | 3 + 6 files changed, 63 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index c488127857ed5..a633cb0787f27 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -60,6 +60,7 @@ public class RemoteFsTranslog extends Translog { private final BooleanSupplier primaryModeSupplier; private final RemoteTranslogTransferTracker remoteTranslogTransferTracker; private volatile long maxRemoteTranslogGenerationUploaded; + private final Object uploadMutex = new Object(); private volatile long minSeqNoToKeep; @@ -237,11 +238,20 @@ public static TranslogTransferManager buildTranslogTransferManager( @Override public boolean ensureSynced(Location location) throws IOException { - try (ReleasableLock ignored = writeLock.acquire()) { - assert location.generation <= current.getGeneration(); - if (location.generation == current.getGeneration()) { - ensureOpen(); - return prepareAndUpload(primaryTermSupplier.getAsLong(), location.generation); + try { + boolean shouldUpload = false; + try (ReleasableLock ignored = writeLock.acquire()) { + assert location.generation <= current.getGeneration(); + if (location.generation == current.getGeneration()) { + ensureOpen(); + if (prepareForUpload(location.generation) == false) { + return false; + } + shouldUpload = true; + } + } + if (shouldUpload) { + return performUpload(primaryTermSupplier.getAsLong(), location.generation); } } catch (final Exception ex) { closeOnTragicEvent(ex); @@ -256,10 +266,12 @@ public void rollGeneration() throws IOException { if (current.totalOperations() == 0 && primaryTermSupplier.getAsLong() == current.getPrimaryTerm()) { return; } - prepareAndUpload(primaryTermSupplier.getAsLong(), null); + if (prepareForUpload(null)) { + performUpload(primaryTermSupplier.getAsLong(), null); + } } - private boolean prepareAndUpload(Long primaryTerm, Long generation) throws IOException { + private boolean prepareForUpload(Long generation) throws IOException { try (Releasable ignored = writeLock.acquire()) { if (generation == null || generation == current.getGeneration()) { try { @@ -275,23 +287,41 @@ private boolean prepareAndUpload(Long primaryTerm, Long generation) throws IOExc closeOnTragicEvent(e); throw e; } - } else if (generation < current.getGeneration()) { - return false; - } + return true; + } else return generation >= current.getGeneration(); + } + } - // Do we need remote writes in sync fashion ? - // If we don't , we should swallow FileAlreadyExistsException while writing to remote store - // and also verify for same during primary-primary relocation - // Writing remote in sync fashion doesn't hurt as global ckp update - // is not updated in remote translog except in primary to primary recovery. - if (generation == null) { - if (closed.get() == false) { - return upload(primaryTerm, current.getGeneration() - 1); + /** + * This method does the remote store upload by first acquiring the lock on the uploadMutex monitor. The synchronized + * is required to restrict multiple uploads happening concurrently. The read lock is required to ensure that the + * underlying translog readers are not deleted and the current writer is not converted to a reader at the time of + * upload. + * + * @param primaryTerm current primary term + * @param generation current generation + * @return true if upload is successful + * @throws IOException if the upload fails due to any underlying exceptions. + */ + private boolean performUpload(Long primaryTerm, Long generation) throws IOException { + synchronized (uploadMutex) { + try (Releasable ignored = readLock.acquire()) { + // Do we need remote writes in sync fashion ? + // If we don't , we should swallow FileAlreadyExistsException while writing to remote store + // and also verify for same during primary-primary relocation + // Writing remote in sync fashion doesn't hurt as global ckp update + // is not updated in remote translog except in primary to primary recovery. + long generationToUpload; + if (generation == null) { + if (closed.get() == false) { + generationToUpload = current.getGeneration() - 1; + } else { + generationToUpload = current.getGeneration(); + } } else { - return upload(primaryTerm, current.getGeneration()); + generationToUpload = generation; } - } else { - return upload(primaryTerm, generation); + return upload(primaryTerm, generationToUpload); } } } @@ -343,8 +373,8 @@ private boolean syncToDisk() throws IOException { @Override public void sync() throws IOException { try { - if (syncToDisk() || syncNeeded()) { - prepareAndUpload(primaryTermSupplier.getAsLong(), null); + if ((syncToDisk() || syncNeeded()) && prepareForUpload(null)) { + performUpload(primaryTermSupplier.getAsLong(), null); } } catch (final Exception e) { tragedy.setTragicException(e); @@ -528,8 +558,6 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen @Override public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOException { - transferReleasable.close(); - closeFilesIfNoPendingRetentionLocks(); maxRemoteTranslogGenerationUploaded = generation; minRemoteGenReferenced = getMinFileGeneration(); logger.trace("uploaded translog for {} {} ", primaryTerm, generation); @@ -537,13 +565,16 @@ public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOExcepti @Override public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) throws IOException { - transferReleasable.close(); - closeFilesIfNoPendingRetentionLocks(); if (ex instanceof IOException) { throw (IOException) ex; } else { throw (RuntimeException) ex; } } + + @Override + public void close() { + transferReleasable.close(); + } } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java index 10dec13c81e1a..eb0eebb564b63 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java @@ -164,7 +164,6 @@ public TranslogCheckpointTransferSnapshot build() throws IOException { translogTransferSnapshot.setMinTranslogGeneration(highestGenMinTranslogGeneration); assert this.primaryTerm == highestGenPrimaryTerm : "inconsistent primary term"; - assert this.generation == highestGeneration : " inconsistent generation "; final long finalHighestGeneration = highestGeneration; assert LongStream.iterate(lowestGeneration, i -> i + 1) .limit(highestGeneration) diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index fd4936603671c..fe6b5dab9937b 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -109,7 +109,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans long prevUploadBytesSucceeded = remoteTranslogTransferTracker.getUploadBytesSucceeded(); long prevUploadTimeInMillis = remoteTranslogTransferTracker.getTotalUploadTimeInMillis(); - try { + try (translogTransferListener) { toUpload.addAll(fileTransferTracker.exclusionFilter(transferSnapshot.getTranslogFileSnapshots())); toUpload.addAll(fileTransferTracker.exclusionFilter((transferSnapshot.getCheckpointFileSnapshots()))); if (toUpload.isEmpty()) { diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java b/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java index 132d1adf916da..8805c16298d96 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java @@ -17,7 +17,7 @@ * * @opensearch.internal */ -public interface TranslogTransferListener { +public interface TranslogTransferListener extends AutoCloseable { /** * Invoked when the transfer of {@link TransferSnapshot} succeeds * @param transferSnapshot the transfer snapshot diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index de1b2990f0a50..3c654818ffc6c 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -108,7 +108,6 @@ import static org.mockito.Mockito.when; @LuceneTestCase.SuppressFileSystems("ExtrasFS") - public class RemoteFsTranslogTests extends OpenSearchTestCase { protected final ShardId shardId = new ShardId("index", "_na_", 1); diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 6fc4557a75675..0987201eb8602 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -168,6 +168,9 @@ public void onUploadComplete(TransferSnapshot transferSnapshot) { public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) { translogTransferFailed.incrementAndGet(); } + + @Override + public void close() {} })); assertEquals(4, fileTransferSucceeded.get()); assertEquals(0, fileTransferFailed.get()); From 5b864c0e023647cf653300d638f60fc0136bf95b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:40:19 -0400 Subject: [PATCH 042/388] Bump com.zaxxer:SparseBitSet from 1.2 to 1.3 in /plugins/ingest-attachment (#10098) * Bump com.zaxxer:SparseBitSet in /plugins/ingest-attachment Bumps [com.zaxxer:SparseBitSet](https://github.com/brettwooldridge/SparseBitSet) from 1.2 to 1.3. - [Commits](https://github.com/brettwooldridge/SparseBitSet/compare/SparseBitSet-1.2...SparseBitSet-1.3) --- updated-dependencies: - dependency-name: com.zaxxer:SparseBitSet dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 3 ++- plugins/ingest-attachment/build.gradle | 2 +- plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1 | 1 - plugins/ingest-attachment/licenses/SparseBitSet-1.3.jar.sha1 | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/SparseBitSet-1.3.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 337573480adfd..d374a725c84ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.cloud:google-cloud-core-http` from 2.21.1 to 2.23.0 ([#9971](https://github.com/opensearch-project/OpenSearch/pull/9971)) - Bump `mockito` from 5.4.0 to 5.5.0 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022)) - Bump `bytebuddy` from 1.14.3 to 1.14.7 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022)) +- Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) @@ -106,4 +107,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index f54c80e9f76c1..330a17c02bc7a 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -93,7 +93,7 @@ dependencies { // Microsoft Word files with visio diagrams api 'org.apache.commons:commons-math3:3.6.1' // POIs dependency - api 'com.zaxxer:SparseBitSet:1.2' + api 'com.zaxxer:SparseBitSet:1.3' } restResources { diff --git a/plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1 b/plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1 deleted file mode 100644 index 5f1d015b87ac7..0000000000000 --- a/plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8467c813d442837fcaeddbc42cf5c5359fab4933 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/SparseBitSet-1.3.jar.sha1 b/plugins/ingest-attachment/licenses/SparseBitSet-1.3.jar.sha1 new file mode 100644 index 0000000000000..2803db7c91e30 --- /dev/null +++ b/plugins/ingest-attachment/licenses/SparseBitSet-1.3.jar.sha1 @@ -0,0 +1 @@ +533eac055afe3d5f614ea95e333afd6c2bde8f26 \ No newline at end of file From 2a5b124ee8ef4376d62c484b6cd3ea1d98ca75d1 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:11:20 +0530 Subject: [PATCH 043/388] Fix cluster chaining during bootstrap (#10020) * Fix clusterUUID chaining logic Signed-off-by: Sooraj Sinha --- .../opensearch/gateway/GatewayMetaState.java | 43 ++++++- .../remote/ClusterMetadataManifest.java | 37 +++++- .../remote/RemoteClusterStateService.java | 100 ++++++++++++--- .../coordination/CoordinationStateTests.java | 36 +++++- .../GatewayMetaStatePersistedStateTests.java | 21 +++- .../remote/ClusterMetadataManifestTests.java | 21 +++- .../RemoteClusterStateServiceTests.java | 117 +++++++++++++++--- 7 files changed, 315 insertions(+), 60 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index e42ac8daa3b1c..6b26af148b2ea 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -78,6 +78,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -171,11 +172,12 @@ public void start( // If the cluster UUID loaded from local is unknown (_na_) then fetch the best state from remote // If there is no valid state on remote, continue with initial empty state // If there is a valid state, then restore index metadata using this state + String lastKnownClusterUUID = ClusterState.UNKNOWN_UUID; if (ClusterState.UNKNOWN_UUID.equals(clusterState.metadata().clusterUUID())) { - String lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote( + lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote( clusterState.getClusterName().value() ); - if (!ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID)) { + if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) { // Load state from remote final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore( clusterState, @@ -186,7 +188,7 @@ public void start( clusterState = remoteRestoreResult.getClusterState(); } } - remotePersistedState = new RemotePersistedState(remoteClusterStateService); + remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID); } persistedState = new LucenePersistedState(persistedClusterStateService, currentTerm, clusterState); } else { @@ -647,9 +649,11 @@ public static class RemotePersistedState implements PersistedState { private ClusterState lastAcceptedState; private ClusterMetadataManifest lastAcceptedManifest; private final RemoteClusterStateService remoteClusterStateService; + private String previousClusterUUID; - public RemotePersistedState(final RemoteClusterStateService remoteClusterStateService) { + public RemotePersistedState(final RemoteClusterStateService remoteClusterStateService, final String previousClusterUUID) { this.remoteClusterStateService = remoteClusterStateService; + this.previousClusterUUID = previousClusterUUID; } @Override @@ -674,7 +678,26 @@ public void setLastAcceptedState(ClusterState clusterState) { try { final ClusterMetadataManifest manifest; if (shouldWriteFullClusterState(clusterState)) { - manifest = remoteClusterStateService.writeFullMetadata(clusterState); + if (clusterState.metadata().clusterUUIDCommitted() == true) { + final Optional latestManifest = remoteClusterStateService.getLatestClusterMetadataManifest( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID() + ); + if (latestManifest.isPresent()) { + // The previous UUID should not change for the current UUID. So fetching the latest manifest + // from remote store and getting the previous UUID. + previousClusterUUID = latestManifest.get().getPreviousClusterUUID(); + } else { + // When the user starts the cluster with remote state disabled but later enables the remote state, + // there will not be any manifest for the current cluster UUID. + logger.error( + "Latest manifest is not present in remote store for cluster UUID: {}", + clusterState.metadata().clusterUUID() + ); + previousClusterUUID = ClusterState.UNKNOWN_UUID; + } + } + manifest = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID); } else { assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) == true : "Previous manifest and previous ClusterState are not in sync"; @@ -723,11 +746,19 @@ public void markLastAcceptedStateAsCommitted() { try { assert lastAcceptedState != null : "Last accepted state is not present"; assert lastAcceptedManifest != null : "Last accepted manifest is not present"; + ClusterState clusterState = lastAcceptedState; + if (lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false + && lastAcceptedState.metadata().clusterUUIDCommitted() == false) { + Metadata.Builder metadataBuilder = Metadata.builder(lastAcceptedState.metadata()); + metadataBuilder.clusterUUIDCommitted(true); + clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build(); + } final ClusterMetadataManifest committedManifest = remoteClusterStateService.markLastStateAsCommitted( - lastAcceptedState, + clusterState, lastAcceptedManifest ); lastAcceptedManifest = committedManifest; + lastAcceptedState = clusterState; } catch (Exception e) { handleExceptionOnWrite(e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 040c0663efbd9..40b16f3d6323b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -42,6 +42,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField COMMITTED_FIELD = new ParseField("committed"); private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); + private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); private static long term(Object[] fields) { return (long) fields[0]; @@ -79,6 +80,10 @@ private static String previousClusterUUID(Object[] fields) { return (String) fields[8]; } + private static boolean clusterUUIDCommitted(Object[] fields) { + return (boolean) fields[9]; + } + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> new ClusterMetadataManifest( @@ -90,7 +95,8 @@ private static String previousClusterUUID(Object[] fields) { nodeId(fields), committed(fields), indices(fields), - previousClusterUUID(fields) + previousClusterUUID(fields), + clusterUUIDCommitted(fields) ) ); @@ -108,6 +114,7 @@ private static String previousClusterUUID(Object[] fields) { INDICES_FIELD ); PARSER.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_COMMITTED); } private final List indices; @@ -119,6 +126,7 @@ private static String previousClusterUUID(Object[] fields) { private final String nodeId; private final boolean committed; private final String previousClusterUUID; + private final boolean clusterUUIDCommitted; public List getIndices() { return indices; @@ -156,6 +164,10 @@ public String getPreviousClusterUUID() { return previousClusterUUID; } + public boolean isClusterUUIDCommitted() { + return clusterUUIDCommitted; + } + public ClusterMetadataManifest( long clusterTerm, long version, @@ -165,7 +177,8 @@ public ClusterMetadataManifest( String nodeId, boolean committed, List indices, - String previousClusterUUID + String previousClusterUUID, + boolean clusterUUIDCommitted ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -176,6 +189,7 @@ public ClusterMetadataManifest( this.committed = committed; this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; + this.clusterUUIDCommitted = clusterUUIDCommitted; } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -188,6 +202,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.committed = in.readBoolean(); this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); + this.clusterUUIDCommitted = in.readBoolean(); } public static Builder builder() { @@ -215,6 +230,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); builder.field(PREVIOUS_CLUSTER_UUID.getPreferredName(), getPreviousClusterUUID()); + builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted()); return builder; } @@ -229,6 +245,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(committed); out.writeCollection(indices); out.writeString(previousClusterUUID); + out.writeBoolean(clusterUUIDCommitted); } @Override @@ -248,7 +265,8 @@ public boolean equals(Object o) { && Objects.equals(opensearchVersion, that.opensearchVersion) && Objects.equals(nodeId, that.nodeId) && Objects.equals(committed, that.committed) - && Objects.equals(previousClusterUUID, that.previousClusterUUID); + && Objects.equals(previousClusterUUID, that.previousClusterUUID) + && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted); } @Override @@ -262,7 +280,8 @@ public int hashCode() { opensearchVersion, nodeId, committed, - previousClusterUUID + previousClusterUUID, + clusterUUIDCommitted ); } @@ -291,6 +310,7 @@ public static class Builder { private String nodeId; private String previousClusterUUID; private boolean committed; + private boolean clusterUUIDCommitted; public Builder indices(List indices) { this.indices = indices; @@ -341,6 +361,11 @@ public Builder previousClusterUUID(String previousClusterUUID) { return this; } + public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { + this.clusterUUIDCommitted = clusterUUIDCommitted; + return this; + } + public Builder() { indices = new ArrayList<>(); } @@ -355,6 +380,7 @@ public Builder(ClusterMetadataManifest manifest) { this.committed = manifest.committed; this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; + this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; } public ClusterMetadataManifest build() { @@ -367,7 +393,8 @@ public ClusterMetadataManifest build() { nodeId, committed, indices, - previousClusterUUID + previousClusterUUID, + clusterUUIDCommitted ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index cf750bb11f3f8..dddc5376803a5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -152,20 +152,13 @@ private BlobStoreTransferService getBlobStoreTransferService() { * @return A manifest object which contains the details of uploaded entity metadata. */ @Nullable - public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState) throws IOException { + public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, String previousClusterUUID) throws IOException { final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { logger.error("Local node is not elected cluster manager. Exiting"); return null; } - // should fetch the previous cluster UUID before writing full cluster state. - // Whenever a new election happens, a new leader will be elected and it might have stale previous UUID - final String previousClusterUUID = fetchPreviousClusterUUID( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() - ); - // any validations before/after upload ? final List allUploadedIndexMetadata = writeIndexMetadataParallel( clusterState, @@ -436,7 +429,8 @@ private ClusterMetadataManifest uploadManifest( nodeId, committed, uploadedIndexMetadata, - previousClusterUUID + previousClusterUUID, + clusterState.metadata().clusterUUIDCommitted() ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; @@ -582,7 +576,7 @@ public String getLastKnownUUIDFromRemote(String clusterName) { try { Set clusterUUIDs = getAllClusterUUIDs(clusterName); Map latestManifests = getLatestManifestForAllClusterUUIDs(clusterName, clusterUUIDs); - List validChain = createClusterChain(latestManifests); + List validChain = createClusterChain(latestManifests, clusterName); if (validChain.isEmpty()) { return ClusterState.UNKNOWN_UUID; } @@ -623,7 +617,7 @@ private Map getLatestManifestForAllClusterUUIDs * @param manifestsByClusterUUID Map of latest ClusterMetadataManifest for every cluster UUID * @return List of cluster UUIDs. The first element is the most recent cluster UUID in the chain */ - private List createClusterChain(final Map manifestsByClusterUUID) { + private List createClusterChain(final Map manifestsByClusterUUID, final String clusterName) { final Map clusterUUIDGraph = manifestsByClusterUUID.values() .stream() .collect(Collectors.toMap(ClusterMetadataManifest::getClusterUUID, ClusterMetadataManifest::getPreviousClusterUUID)); @@ -637,18 +631,29 @@ private List createClusterChain(final Map 1) { - throw new IllegalStateException( - String.format( - Locale.ROOT, - "The system has ended into multiple valid cluster states in the remote store. " - + "Please check their latest manifest to decide which one you want to keep. Valid Cluster UUIDs: - %s", - validClusterUUIDs - ) + // If the valid cluster UUIDs are more that 1, it means there was some race condition where + // more then 2 cluster manager nodes tried to become active cluster manager and published + // 2 cluster UUIDs which followed the same previous UUID. + final Map manifestsByClusterUUIDTrimmed = trimClusterUUIDs( + manifestsByClusterUUID, + validClusterUUIDs, + clusterName ); + if (manifestsByClusterUUID.size() == manifestsByClusterUUIDTrimmed.size()) { + throw new IllegalStateException( + String.format( + Locale.ROOT, + "The system has ended into multiple valid cluster states in the remote store. " + + "Please check their latest manifest to decide which one you want to keep. Valid Cluster UUIDs: - %s", + validClusterUUIDs + ) + ); + } + return createClusterChain(manifestsByClusterUUIDTrimmed, clusterName); } final List validChain = new ArrayList<>(); String currentUUID = validClusterUUIDs.get(0); - while (!ClusterState.UNKNOWN_UUID.equals(currentUUID)) { + while (currentUUID != null && !ClusterState.UNKNOWN_UUID.equals(currentUUID)) { validChain.add(currentUUID); // Getting the previous cluster UUID of a cluster UUID from the clusterUUID Graph currentUUID = clusterUUIDGraph.get(currentUUID); @@ -656,8 +661,61 @@ private List createClusterChain(final Map trimClusterUUIDs( + final Map latestManifestsByClusterUUID, + final List validClusterUUIDs, + final String clusterName + ) { + final Map trimmedUUIDs = new HashMap<>(latestManifestsByClusterUUID); + for (String clusterUUID : validClusterUUIDs) { + ClusterMetadataManifest currentManifest = trimmedUUIDs.get(clusterUUID); + // Here we compare the manifest of current UUID to that of previous UUID + // In case currentUUID's latest manifest is same as previous UUIDs latest manifest, + // that means it was restored from previousUUID and no IndexMetadata update was performed on it. + if (ClusterState.UNKNOWN_UUID.equals(currentManifest.getPreviousClusterUUID())) { + if (currentManifest.getIndices().isEmpty()) { + trimmedUUIDs.remove(clusterUUID); + } + } else { + ClusterMetadataManifest previousManifest = trimmedUUIDs.get(currentManifest.getPreviousClusterUUID()); + if (isMetadataEqual(currentManifest, previousManifest, clusterName)) { + trimmedUUIDs.remove(clusterUUID); + } + } + } + return trimmedUUIDs; + } + + private boolean isMetadataEqual(ClusterMetadataManifest first, ClusterMetadataManifest second, String clusterName) { + // todo clusterName can be set as final in the constructor + if (first.getIndices().size() != second.getIndices().size()) { + return false; + } + final Map secondIndices = second.getIndices() + .stream() + .collect(Collectors.toMap(md -> md.getIndexName(), Function.identity())); + for (UploadedIndexMetadata uploadedIndexMetadata : first.getIndices()) { + final IndexMetadata firstIndexMetadata = getIndexMetadata(clusterName, first.getClusterUUID(), uploadedIndexMetadata); + final UploadedIndexMetadata secondUploadedIndexMetadata = secondIndices.get(uploadedIndexMetadata.getIndexName()); + if (secondUploadedIndexMetadata == null) { + return false; + } + final IndexMetadata secondIndexMetadata = getIndexMetadata(clusterName, second.getClusterUUID(), secondUploadedIndexMetadata); + if (firstIndexMetadata.equals(secondIndexMetadata) == false) { + return false; + } + } + return true; + } + private boolean isInvalidClusterUUID(ClusterMetadataManifest manifest) { - return !manifest.isCommitted() && manifest.getIndices().isEmpty(); + return !manifest.isClusterUUIDCommitted(); } /** @@ -729,6 +787,7 @@ public IndexMetadataTransferException(String errorDesc, Throwable cause) { /** * Purges all remote cluster state against provided cluster UUIDs + * * @param clusterName name of the cluster * @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged */ @@ -760,6 +819,7 @@ public void onFailure(Exception e) { /** * Deletes older than last {@code versionsToRetain} manifests. Also cleans up unreferenced IndexMetadata associated with older manifests + * * @param clusterName name of the cluster * @param clusterUUID uuid of cluster state to refer to in remote * @param manifestsToRetain no of latest manifest files to keep in remote diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index d1c2dda615992..f37823d2c0c7d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -60,6 +60,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import static java.util.Collections.emptyMap; @@ -70,6 +71,9 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class CoordinationStateTests extends OpenSearchTestCase { @@ -925,6 +929,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final VotingConfiguration initialConfig = VotingConfiguration.of(node1); final ClusterState clusterState = clusterState(0L, 0L, node1, initialConfig, initialConfig, 42L); + final String previousClusterUUID = "prev-cluster-uuid"; final ClusterMetadataManifest manifest = new ClusterMetadataManifest( 0L, 0L, @@ -934,13 +939,17 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep randomAlphaOfLength(10), false, Collections.emptyList(), - randomAlphaOfLength(10) + randomAlphaOfLength(10), + true ); - Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState)).thenReturn(manifest); + Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, ps1); - persistedStateRegistry.addPersistedState(PersistedStateType.REMOTE, new RemotePersistedState(remoteClusterStateService)); + persistedStateRegistry.addPersistedState( + PersistedStateType.REMOTE, + new RemotePersistedState(remoteClusterStateService, previousClusterUUID) + ); String randomRepoName = "randomRepoName"; String stateRepoTypeAttributeKey = String.format( @@ -963,11 +972,28 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep final CoordinationState coordinationState = createCoordinationState(persistedStateRegistry, node1, settings); coordinationState.handlePrePublish(clusterState); - Mockito.verify(remoteClusterStateService, Mockito.times(1)).writeFullMetadata(clusterState); + Mockito.verify(remoteClusterStateService, Mockito.times(1)).writeFullMetadata(clusterState, previousClusterUUID); assertThat(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState(), equalTo(clusterState)); coordinationState.handlePreCommit(); - Mockito.verify(remoteClusterStateService, Mockito.times(1)).markLastStateAsCommitted(clusterState, manifest); + ClusterState committedClusterState = ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).clusterUUIDCommitted(true).build()) + .build(); + // Mockito.verify(remoteClusterStateService, Mockito.times(1)).markLastStateAsCommitted(committedClusterState, manifest); + ArgumentCaptor clusterStateCaptor = ArgumentCaptor.forClass(ClusterState.class); + verify(remoteClusterStateService, times(1)).markLastStateAsCommitted(clusterStateCaptor.capture(), any()); + assertThat(clusterStateCaptor.getValue().metadata().indices(), equalTo(committedClusterState.metadata().indices())); + assertThat(clusterStateCaptor.getValue().metadata().clusterUUID(), equalTo(committedClusterState.metadata().clusterUUID())); + assertThat(clusterStateCaptor.getValue().stateUUID(), equalTo(committedClusterState.stateUUID())); + assertThat( + clusterStateCaptor.getValue().coordinationMetadata().term(), + equalTo(committedClusterState.coordinationMetadata().term()) + ); + assertThat(clusterStateCaptor.getValue().version(), equalTo(committedClusterState.version())); + assertThat( + clusterStateCaptor.getValue().metadata().clusterUUIDCommitted(), + equalTo(committedClusterState.metadata().clusterUUIDCommitted()) + ); } public static CoordinationState createCoordinationState( diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 486717faaf864..c7ed1cb732154 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -716,10 +716,11 @@ Directory createDirectory(Path path) { public void testRemotePersistedState() throws IOException { final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder().clusterTerm(1L).stateVersion(5L).build(); - Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any())).thenReturn(manifest); + final String previousClusterUUID = "prev-cluster-uuid"; + Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())).thenReturn(manifest); Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(manifest); - CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService); + CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID); assertThat(remotePersistedState.getLastAcceptedState(), nullValue()); assertThat(remotePersistedState.getCurrentTerm(), equalTo(0L)); @@ -731,7 +732,7 @@ public void testRemotePersistedState() throws IOException { ); remotePersistedState.setLastAcceptedState(clusterState); - Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState); + Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState, previousClusterUUID); assertThat(remotePersistedState.getLastAcceptedState(), equalTo(clusterState)); assertThat(remotePersistedState.getCurrentTerm(), equalTo(clusterTerm)); @@ -742,7 +743,7 @@ public void testRemotePersistedState() throws IOException { ); remotePersistedState.setLastAcceptedState(secondClusterState); - Mockito.verify(remoteClusterStateService, times(1)).writeFullMetadata(secondClusterState); + Mockito.verify(remoteClusterStateService, times(1)).writeFullMetadata(secondClusterState, previousClusterUUID); assertThat(remotePersistedState.getLastAcceptedState(), equalTo(secondClusterState)); assertThat(remotePersistedState.getCurrentTerm(), equalTo(clusterTerm)); @@ -752,14 +753,22 @@ public void testRemotePersistedState() throws IOException { assertThat(remotePersistedState.getLastAcceptedState(), equalTo(secondClusterState)); assertThat(remotePersistedState.getCurrentTerm(), equalTo(clusterTerm)); + assertThat(remotePersistedState.getLastAcceptedState().metadata().clusterUUIDCommitted(), equalTo(false)); + final ClusterState thirdClusterState = ClusterState.builder(secondClusterState) + .metadata(Metadata.builder(secondClusterState.getMetadata()).clusterUUID(randomAlphaOfLength(10)).build()) + .build(); + remotePersistedState.setLastAcceptedState(thirdClusterState); + remotePersistedState.markLastAcceptedStateAsCommitted(); + assertThat(remotePersistedState.getLastAcceptedState().metadata().clusterUUIDCommitted(), equalTo(true)); } public void testRemotePersistedStateExceptionOnFullStateUpload() throws IOException { final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); - Mockito.doThrow(IOException.class).when(remoteClusterStateService).writeFullMetadata(Mockito.any()); + final String previousClusterUUID = "prev-cluster-uuid"; + Mockito.doThrow(IOException.class).when(remoteClusterStateService).writeFullMetadata(Mockito.any(), Mockito.any()); - CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService); + CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID); final long clusterTerm = randomNonNegativeLong(); final ClusterState clusterState = createClusterState( diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 9f8dde5ba9d45..66426c2a880a3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -37,7 +37,8 @@ public void testClusterMetadataManifestXContent() throws IOException { "test-node-id", false, Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid" + "prev-cluster-uuid", + true ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -60,7 +61,8 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "B10RX1f5RJenMQvYccCgSQ", true, randomUploadedIndexMetadataList(), - "yfObdx8KSMKKrXf8UyHhM" + "yfObdx8KSMKKrXf8UyHhM", + true ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -183,6 +185,21 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { ); } + { // Mutate cluster uuid committed + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.clusterUUIDCommitted(false); + return builder.build(); + } + ); + } } private List randomUploadedIndexMetadataList() { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 9f5067420aab1..65166386733c6 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -135,7 +135,7 @@ public void teardown() throws Exception { public void testFailWriteFullMetadataNonClusterManagerNode() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().build(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)); Assert.assertThat(manifest, nullValue()); } @@ -169,7 +169,7 @@ public void testWriteFullMetadataSuccess() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); List indices = List.of(uploadedIndexMetadata); @@ -190,6 +190,7 @@ public void testWriteFullMetadataSuccess() throws IOException { assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); } public void testWriteFullMetadataInParallelSuccess() throws IOException { @@ -205,7 +206,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { }).when(container).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture()); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); List indices = List.of(uploadedIndexMetadata); @@ -216,6 +217,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { .stateVersion(1L) .stateUUID("state-uuid") .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") .build(); assertThat(manifest.getIndices().size(), is(1)); @@ -226,6 +228,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); assertEquals(actionListenerArgumentCaptor.getAllValues().size(), 1); assertEquals(writeContextArgumentCaptor.getAllValues().size(), 1); @@ -266,7 +269,7 @@ public void testWriteFullMetadataInParallelFailure() throws IOException { remoteClusterStateService.start(); assertThrows( RemoteClusterStateService.IndexMetadataTransferException.class, - () -> remoteClusterStateService.writeFullMetadata(clusterState) + () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)) ); } @@ -571,12 +574,43 @@ public void testGetValidPreviousClusterUUID() throws IOException { public void testGetValidPreviousClusterUUIDForInvalidChain() throws IOException { Map clusterUUIDsPointers = Map.of( + "cluster-uuid2", "cluster-uuid1", - ClusterState.UNKNOWN_UUID, + "cluster-uuid3", + "cluster-uuid2", + "cluster-uuid5", + "cluster-uuid4" + ); + mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers); + + remoteClusterStateService.start(); + assertThrows(IllegalStateException.class, () -> remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")); + } + + public void testGetValidPreviousClusterUUIDWithMultipleChains() throws IOException { + Map clusterUUIDsPointers = Map.of( "cluster-uuid2", + "cluster-uuid1", + "cluster-uuid1", ClusterState.UNKNOWN_UUID, "cluster-uuid3", - "cluster-uuid2" + "cluster-uuid1" + ); + mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers); + + remoteClusterStateService.start(); + String previousClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster"); + assertThat(previousClusterUUID, equalTo("cluster-uuid3")); + } + + public void testGetValidPreviousClusterUUIDWithInvalidMultipleChains() throws IOException { + Map clusterUUIDsPointers = Map.of( + "cluster-uuid1", + ClusterState.UNKNOWN_UUID, + "cluster-uuid2", + "cluster-uuid1", + "cluster-uuid3", + ClusterState.UNKNOWN_UUID ); mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers); @@ -598,42 +632,92 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste when(blobContainer3.path()).thenReturn(blobPath); mockBlobContainerForClusterUUIDs(uuidBlobContainer, clusterUUIDsPointers.keySet()); + List uploadedIndexMetadataList1 = List.of( + new UploadedIndexMetadata("index1", "index-uuid1", "key1"), + new UploadedIndexMetadata("index2", "index-uuid2", "key2") + ); final ClusterMetadataManifest clusterManifest1 = generateClusterMetadataManifest( "cluster-uuid1", clusterUUIDsPointers.get("cluster-uuid1"), - randomAlphaOfLength(10) + randomAlphaOfLength(10), + uploadedIndexMetadataList1 ); - mockBlobContainer(blobContainer1, clusterManifest1, Map.of()); + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexMetadata indexMetadata1 = IndexMetadata.builder("index1") + .settings(indexSettings) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + IndexMetadata indexMetadata2 = IndexMetadata.builder("index2") + .settings(indexSettings) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + Map indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2); + mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1); + List uploadedIndexMetadataList2 = List.of( + new UploadedIndexMetadata("index1", "index-uuid1", "key1"), + new UploadedIndexMetadata("index2", "index-uuid2", "key2") + ); final ClusterMetadataManifest clusterManifest2 = generateClusterMetadataManifest( "cluster-uuid2", clusterUUIDsPointers.get("cluster-uuid2"), - randomAlphaOfLength(10) + randomAlphaOfLength(10), + uploadedIndexMetadataList2 ); - mockBlobContainer(blobContainer2, clusterManifest2, Map.of()); + IndexMetadata indexMetadata3 = IndexMetadata.builder("index1") + .settings(indexSettings) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + IndexMetadata indexMetadata4 = IndexMetadata.builder("index2") + .settings(indexSettings) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + Map indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4); + mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2); + List uploadedIndexMetadataList3 = List.of(new UploadedIndexMetadata("index1", "index-uuid1", "key1")); final ClusterMetadataManifest clusterManifest3 = generateClusterMetadataManifest( "cluster-uuid3", clusterUUIDsPointers.get("cluster-uuid3"), - randomAlphaOfLength(10) + randomAlphaOfLength(10), + uploadedIndexMetadataList3 ); - mockBlobContainer(blobContainer3, clusterManifest3, Map.of()); + IndexMetadata indexMetadata5 = IndexMetadata.builder("index1") + .settings(indexSettings) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + Map indexMetadataMap3 = Map.of("index-uuid1", indexMetadata5); + mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3); when(blobStore.blobContainer(ArgumentMatchers.any())).thenReturn( uuidBlobContainer, blobContainer1, blobContainer1, + blobContainer3, + blobContainer3, blobContainer2, blobContainer2, - blobContainer3, - blobContainer3 + blobContainer1, + blobContainer2, + blobContainer1, + blobContainer2 ); when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); } - private ClusterMetadataManifest generateClusterMetadataManifest(String clusterUUID, String previousClusterUUID, String stateUUID) { + private ClusterMetadataManifest generateClusterMetadataManifest( + String clusterUUID, + String previousClusterUUID, + String stateUUID, + List uploadedIndexMetadata + ) { return ClusterMetadataManifest.builder() - .indices(List.of()) + .indices(uploadedIndexMetadata) .clusterTerm(1L) .stateVersion(1L) .stateUUID(stateUUID) @@ -642,6 +726,7 @@ private ClusterMetadataManifest generateClusterMetadataManifest(String clusterUU .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID(previousClusterUUID) .committed(true) + .clusterUUIDCommitted(true) .build(); } From 256ea418015c513a96390407ab0ddfa5704269b2 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 19 Sep 2023 14:47:11 -0400 Subject: [PATCH 044/388] Update Apache Lucene to 9.8.0-snapshot-95cdd2e (#10031) Signed-off-by: Andriy Redko --- buildSrc/version.properties | 2 +- ...lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...expressions-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...expressions-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...nalysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...nalysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...is-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...is-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...alysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...alysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...is-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...is-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...sis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...sis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...sis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...sis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + .../mapper/CorrelationVectorFieldMapper.java | 5 ++--- .../query/CorrelationQueryBuilderTests.java | 2 +- ...ysis-common-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...ysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...ward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...ward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...ne-grouping-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...ne-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...highlighter-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...cene-memory-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...cene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...ene-queries-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...ene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...queryparser-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...ene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...ene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...tial-extras-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...tial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...e-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...e-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + ...ene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 | 1 - ...ene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 + .../node/tasks/ConcurrentSearchTasksIT.java | 5 +++-- .../index/engine/TranslogLeafReader.java | 6 +++--- .../comparators/UnsignedLongComparator.java | 18 ++++++++++-------- 52 files changed, 43 insertions(+), 41 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index be09584b1dba1..4a31582bb7546 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.8.0-snapshot-4373c3b +lucene = 9.8.0-snapshot-95cdd2e bundled_jdk_vendor = adoptium bundled_jdk = 20.0.2+9 diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index dc363f2776429..0000000000000 --- a/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -30c3afcf058532d3d2b8820375043000e7f34a9b \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..70baf1270cd5d --- /dev/null +++ b/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +d2f7fbc5b2c49ca777a169d579f41082a9a57cc7 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 6eaa40708e4ae..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9f8a34fc3d450343ab05ccb5af318a836a6a5fb3 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..c1daa91dd5433 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +57e2b0cca55da8ad856dfd60be42e6daabbc98c3 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 8a3332c950b6d..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fde64e3b23bc9a0849b9897febfe9f13c5113143 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..035b47c5f388c --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +0deb3b85eadf831be17b48acab0785fd9d34fc44 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 33c2afacf2395..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b01a791705fa01fce48dd02ea79fa8045de8dd5e \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..6ff5a433f0a4e --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +9a204267d68ce4ba36bfddc366cd6865cf5e1378 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 1e7986dafa11e..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -43d19320b1b9cd18638b1602fa87d5f21ee043bc \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..a65ab33a31e2a --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +71e8e811f873ba2b47c7ecf9d890cbeac5b6be41 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 14880d9c2d243..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9244dc232f175010b480d4d88e13945c17a0b28b \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..04ab7b7e7adb8 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +6e1274273895365bd83391cc4b79f5264479f5de \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index edc4de3fffe28..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3101a4f79820c1ca3dfb8f49b74c5fb5b32940e1 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..cef3f97d03c51 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +e634c8685edad2bdb5c13748b18c0c1a46bb63a3 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 54c310277b09b..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f12b2a22cd5ebcd84f40a40e78fdd4e268b3b26d \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..3e2dd19a9dd85 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +0afdf2afacbae39414ed06325fbb4bed17c07a7d \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 358db9ea3f0f5..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7dbf5cc3dff93cc1ffe45d79b129859590d001dd \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..8c0544acd1ca0 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +166e2ea297182f7bf7070af02aacea9e6a3a19c8 \ No newline at end of file diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapper.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapper.java index a1918f3c954d0..18c9dd222e2cf 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapper.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapper.java @@ -8,6 +8,7 @@ package org.opensearch.plugin.correlation.core.index.mapper; +import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.document.StoredField; @@ -23,8 +24,6 @@ import java.util.Locale; import java.util.Optional; -import static org.apache.lucene.index.FloatVectorValues.MAX_DIMENSIONS; - /** * Field mapper for the correlation vector type * @@ -32,7 +31,7 @@ */ public class CorrelationVectorFieldMapper extends VectorFieldMapper { - private static final int LUCENE_MAX_DIMENSION = MAX_DIMENSIONS; + private static final int LUCENE_MAX_DIMENSION = KnnVectorsFormat.DEFAULT_MAX_DIMENSIONS; private final FieldType vectorFieldType; diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java index b3fcbedd74558..134c6519d7220 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java @@ -203,7 +203,7 @@ public void testDoToQueryInvalidFieldType() { /** * test serialization of Correlation Query Builder - * @throws Exception + * @throws Exception Exception */ public void testSerialization() throws Exception { assertSerialization(Optional.empty()); diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index e7c7dc2bbc046..0000000000000 --- a/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1446b7641743a1082b566179d1bf2960f5a0724b \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..4ac89f2e792d7 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +8c82be3d997d781bb72d6d0eadade064dd2cd6db \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index d0f64519cd6ff..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -127032ea137d2501b24f0e35e5f9a2e1c7864633 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..624b5174a444f --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +4c261d17c681c0d91171c67e192abfef59adea2e \ No newline at end of file diff --git a/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index dc363f2776429..0000000000000 --- a/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -30c3afcf058532d3d2b8820375043000e7f34a9b \ No newline at end of file diff --git a/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..70baf1270cd5d --- /dev/null +++ b/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +d2f7fbc5b2c49ca777a169d579f41082a9a57cc7 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 294beba43f62a..0000000000000 --- a/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6f742efe0ef3b383468fe38f88ab2dd69ed3d2c \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..20ddb9ae3ef27 --- /dev/null +++ b/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +8d1cf3d6db43fad6630376ba59451f848f4d387c \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index c2a2ef5b13946..0000000000000 --- a/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3162856444777130dee2c4cabe1bf6d18710ff63 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..c3ad03ca53b13 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +83ab97638bb5269f950d75bba5675d3cfb63f2fa \ No newline at end of file diff --git a/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 7c6adaaba9cf1..0000000000000 --- a/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5fe8383516eca7300f978ce38042e327b0a57877 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..c2a4c5334b314 --- /dev/null +++ b/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +97c26362151908dc892263edda3872abbacb71a8 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 586702c968a77..0000000000000 --- a/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b3e77970485be6d2dd59b999bbaa65a2cb993744 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..32534d07e47dc --- /dev/null +++ b/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +8337eddc0dddd0d7dd50c5aa0d17e5e31592f9fa \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 493598eefff5e..0000000000000 --- a/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -86d667ea2f7fb2142d2acacf801dcea47d014a5e \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..7db245cc521c7 --- /dev/null +++ b/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +a2e3fae930295f0e2b401effe04eafc25692a414 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 1bf937f10d795..0000000000000 --- a/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -930d004de698f374da8ac5530fd80e241edeba45 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..d01a6d733196e --- /dev/null +++ b/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +e88d8a464e6cfa345b946c9c8822ba7ee2a9159f \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 74458bc93f90b..0000000000000 --- a/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f62882823d5aa9ed4cf0081a8c18f35e21992080 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..c7b9640bad170 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +9905790675c01e8dc24f9a5e6b9b28b879c65a52 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 3231d0e067940..0000000000000 --- a/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1ec1527e283b423b7ff5e12cd8d889e7247199d \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..c4cd9e47624f8 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +d6c8be427ec8ffc7e8233ffbf0d190d95a56cf14 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index dd47faf91f206..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -de787c052879893e47d21fa161c93413665d55d7 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..dfee145d3ea26 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +11716d61288feaa692593bf699affa8de2b564c4 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 2b378438bfb14..0000000000000 --- a/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e541ed960a571f5d9a0ecff5c26fd5ca857581e \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..c7410086ba86c --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +3a888e06c0535403b9e58a8dcddeb5e6513a4930 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 deleted file mode 100644 index 1e3ed6561e3ef..0000000000000 --- a/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b222ef09a5f20896d031a8322f2e69304c16384 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 new file mode 100644 index 0000000000000..6d8d4205f4d02 --- /dev/null +++ b/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 @@ -0,0 +1 @@ +52dfc8bf135ed29f5baf0a967c1bb63dedb9a069 \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java index ceacb028698de..c733329a1b5f7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java @@ -108,8 +108,9 @@ public void testConcurrentSearchTaskTracking() { assertEquals(mainTaskInfo.getTaskId(), taskInfo.getParentTaskId()); Map> threadStats = getThreadStats(SearchAction.NAME + "[*]", taskInfo.getTaskId()); - // Concurrent search forks each slice of 5 segments to different thread - assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0), threadStats.size()); + // Concurrent search forks each slice of 5 segments to different thread (see please + // https://github.com/apache/lucene/issues/12498) + assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0) + 1, threadStats.size()); // assert that all task descriptions have non-zero length MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0)); diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java index 417cdd5a8f030..c1f69d1ef3638 100644 --- a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java +++ b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java @@ -52,7 +52,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.opensearch.common.util.set.Sets; @@ -271,12 +271,12 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector k, Bits acceptDocs) throws IOException { throw new UnsupportedOperationException(); } @Override - public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector k, Bits acceptDocs) throws IOException { throw new UnsupportedOperationException(); } } diff --git a/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java b/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java index 78b4a5f04c955..d46b34fe97356 100644 --- a/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java +++ b/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java @@ -86,14 +86,6 @@ public void copy(int slot, int doc) throws IOException { super.copy(slot, doc); } - @Override - protected boolean isMissingValueCompetitive() { - int result = missingValue.compareTo(bottom); - // in reverse (desc) sort missingValue is competitive when it's greater or equal to bottom, - // in asc sort missingValue is competitive when it's smaller or equal to bottom - return reverse ? (result >= 0) : (result <= 0); - } - @Override protected void encodeBottom(byte[] packedValue) { BigIntegerPoint.encodeDimension(bottom, packedValue, 0); @@ -103,5 +95,15 @@ protected void encodeBottom(byte[] packedValue) { protected void encodeTop(byte[] packedValue) { BigIntegerPoint.encodeDimension(topValue, packedValue, 0); } + + @Override + protected int compareMissingValueWithBottomValue() { + return missingValue.compareTo(bottom); + } + + @Override + protected int compareMissingValueWithTopValue() { + return missingValue.compareTo(topValue); + } } } From ed1c7ea58f54e480d21e472a5eb53961d1a489c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 16:09:49 -0700 Subject: [PATCH 045/388] Bump tibdex/github-app-token from 1.5.0 to 2.1.0 (#10125) * Bump tibdex/github-app-token from 1.5.0 to 2.1.0 Bumps [tibdex/github-app-token](https://github.com/tibdex/github-app-token) from 1.5.0 to 2.1.0. - [Release notes](https://github.com/tibdex/github-app-token/releases) - [Commits](https://github.com/tibdex/github-app-token/compare/v1.5.0...v2.1.0) --- updated-dependencies: - dependency-name: tibdex/github-app-token dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/auto-release.yml | 2 +- .github/workflows/backport.yml | 2 +- .github/workflows/create-documentation-issue.yml | 2 +- .github/workflows/dependabot_pr.yml | 2 +- .github/workflows/version.yml | 2 +- CHANGELOG.md | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index ae41ae00d018d..252cbda1392f8 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -14,7 +14,7 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@v2.1.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 607c6de58b7f6..2a95177174e9b 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -26,7 +26,7 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@v2.1.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} diff --git a/.github/workflows/create-documentation-issue.yml b/.github/workflows/create-documentation-issue.yml index 373eeb3569a51..df63847f8afca 100644 --- a/.github/workflows/create-documentation-issue.yml +++ b/.github/workflows/create-documentation-issue.yml @@ -14,7 +14,7 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@v2.1.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml index f03a3d125067d..2a5e539b214d3 100644 --- a/.github/workflows/dependabot_pr.yml +++ b/.github/workflows/dependabot_pr.yml @@ -11,7 +11,7 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@v2.1.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 2fe4fb1ac9d48..df785bcc70014 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -13,7 +13,7 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@v2.1.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d374a725c84ee..9cb5d39c1a764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `mockito` from 5.4.0 to 5.5.0 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022)) - Bump `bytebuddy` from 1.14.3 to 1.14.7 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022)) - Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098)) +- Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) From 2f7969affddcd4aff41e07b2761d256373667c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Vl=C4=8Dek?= Date: Wed, 20 Sep 2023 01:25:49 +0200 Subject: [PATCH 046/388] Remove outdated default index mapping error message (#7008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This error message is no longer relevant and should not be used now. Closes: #7007 Signed-off-by: Lukáš Vlček --- .../main/java/org/opensearch/index/mapper/MapperService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index 93928a464b138..ecb965061a828 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -172,9 +172,6 @@ public enum MergeReason { ); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(MapperService.class); - static final String DEFAULT_MAPPING_ERROR_MESSAGE = "[_default_] mappings are not allowed on new indices and should no " - + "longer be used. See [https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html" - + "#default-mapping-not-allowed] for more information."; private final IndexAnalyzers indexAnalyzers; From 9b5bf5fad4ba1463742ba1b4d89cbe81ff2e8fbf Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 20 Sep 2023 14:42:07 -0400 Subject: [PATCH 047/388] Update Github action for Gradle precommit on MacOS (#10140) Signed-off-by: Andriy Redko --- .github/workflows/precommit.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index c628f48e57eba..f4622859916c7 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -22,10 +22,6 @@ jobs: - name: Setup docker (missing on MacOS) if: runner.os == 'macos' run: | - # Workaround for https://github.com/actions/runner-images/issues/8104 - brew remove --ignore-dependencies qemu - curl -o ./qemu.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f88e30b3a23ef3735580f9b05535ce5a0a03c9e3/Formula/qemu.rb - brew install ./qemu.rb brew install docker colima start sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock From 0c4e95095e4d44540ea8a90aa5ca55fea1248366 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Wed, 20 Sep 2023 14:49:18 -0400 Subject: [PATCH 048/388] Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 (#9822) * Bumps jetty version to 12.0.1 to fix GMS-2023-1857 Signed-off-by: Darshit Chanpura * Adds this PR to changelog Signed-off-by: Darshit Chanpura * Updates javax websocket server version Signed-off-by: Darshit Chanpura * Updates jetty version to match jetty websocket version Signed-off-by: Darshit Chanpura --------- Signed-off-by: Darshit Chanpura --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cb5d39c1a764..aef0f2ef03496 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963)) +- Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 42de21875d291..5217089cf3b4c 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -33,7 +33,7 @@ apply plugin: 'opensearch.java' group = 'hdfs' versions << [ - 'jetty': '9.4.51.v20230217' + 'jetty': '9.4.52.v20230823' ] dependencies { From b944194d08aa8635369038c94e9084bf87a3c6aa Mon Sep 17 00:00:00 2001 From: sahil <61558528+buddharajusahil@users.noreply.github.com> Date: Wed, 20 Sep 2023 14:41:44 -0700 Subject: [PATCH 049/388] Search latency tracking - Coordinator node (#8386) We are trying to add search stats to coordinator level node stats. This keeps track of the total time, current requests, and total requests of each request phase. Also added support for general coordinator stats as well on the node level. Signed-off-by: sahil buddharaju Signed-off-by: sahil <61558528+buddharajusahil@users.noreply.github.com> Signed-off-by: Sagar Upadhyaya Signed-off-by: Sagar <99425694+sgup432@users.noreply.github.com> Co-authored-by: sahil buddharaju Co-authored-by: Sagar Upadhyaya Co-authored-by: Sagar Upadhyaya Co-authored-by: Sagar <99425694+sgup432@users.noreply.github.com> --- CHANGELOG.md | 1 + .../search/SearchWeightedRoutingIT.java | 31 ++ .../search/stats/SearchStatsIT.java | 35 ++- .../search/AbstractSearchAsyncAction.java | 35 ++- .../search/CanMatchPreFilterSearchPhase.java | 6 +- .../SearchDfsQueryThenFetchAsyncAction.java | 8 +- .../opensearch/action/search/SearchPhase.java | 12 +- .../action/search/SearchPhaseContext.java | 2 + .../action/search/SearchPhaseName.java | 1 + .../SearchQueryThenFetchAsyncAction.java | 8 +- .../SearchRequestOperationsListener.java | 77 +++++ .../action/search/SearchRequestStats.java | 75 +++++ .../action/search/TransportSearchAction.java | 62 +++- .../common/metrics/CounterMetric.java | 1 + .../opensearch/common/metrics/MeanMetric.java | 1 + .../common/settings/ClusterSettings.java | 1 + .../index/search/stats/SearchStats.java | 136 ++++++++- .../opensearch/indices/IndicesService.java | 7 +- .../opensearch/indices/NodeIndicesStats.java | 7 +- .../main/java/org/opensearch/node/Node.java | 5 + .../cluster/node/stats/NodeStatsTests.java | 4 +- .../AbstractSearchAsyncActionTests.java | 273 +++++++++++++++++- .../CanMatchPreFilterSearchPhaseTests.java | 18 +- .../action/search/MockSearchPhaseContext.java | 5 + .../action/search/SearchAsyncActionTests.java | 16 +- .../SearchQueryThenFetchAsyncActionTests.java | 4 +- .../SearchRequestOperationsListenerTests.java | 69 +++++ .../search/SearchRequestStatsTests.java | 150 ++++++++++ .../index/search/stats/SearchStatsTests.java | 41 ++- .../indices/NodeIndicesStatsTests.java | 6 +- .../snapshots/SnapshotResiliencyTests.java | 4 +- 31 files changed, 1052 insertions(+), 49 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java create mode 100644 server/src/main/java/org/opensearch/action/search/SearchRequestStats.java create mode 100644 server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java create mode 100644 server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index aef0f2ef03496..10205ae20c001 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Add coordinator level stats for search latency ([#8386](https://github.com/opensearch-project/OpenSearch/issues/8386)) - Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681)) - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694)) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java index 6fafdb0912470..5207dab83f1d9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java @@ -16,6 +16,7 @@ import org.opensearch.action.get.MultiGetRequest; import org.opensearch.action.get.MultiGetResponse; import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchPhaseName; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -56,9 +57,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.opensearch.action.search.TransportSearchAction.SEARCH_REQUEST_STATS_ENABLED_KEY; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, minNumDataNodes = 3) @@ -74,6 +77,7 @@ public void testSearchWithWRRShardRouting() throws IOException { .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone" + ".values", "a,b,c") .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone") .put("cluster.routing.weighted.fail_open", false) + .put(SEARCH_REQUEST_STATS_ENABLED_KEY, true) .build(); logger.info("--> starting 6 nodes on different zones"); @@ -180,12 +184,39 @@ public void testSearchWithWRRShardRouting() throws IOException { assertFalse(!hitNodes.contains(nodeId)); } nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet(); + int num = 0; + int coordNumber = 0; for (NodeStats stat : nodeStats.getNodes()) { SearchStats.Stats searchStats = stat.getIndices().getSearch().getTotal(); + if (searchStats.getRequestStatsLongHolder() + .getRequestStatsHolder() + .get(SearchPhaseName.QUERY.getName()) + .getTimeInMillis() > 0) { + assertThat( + searchStats.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.QUERY.getName()).getTotal(), + greaterThan(0L) + ); + assertThat( + searchStats.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.FETCH.getName()).getTimeInMillis(), + greaterThan(0L) + ); + assertThat( + searchStats.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.FETCH.getName()).getTotal(), + greaterThan(0L) + ); + assertThat( + searchStats.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.EXPAND.getName()).getTotal(), + greaterThan(0L) + ); + coordNumber += 1; + } Assert.assertTrue(searchStats.getQueryCount() > 0L); Assert.assertTrue(searchStats.getFetchCount() > 0L); + num++; } + assertThat(coordNumber, greaterThan(0)); + assertThat(num, greaterThan(0)); } private Map> setupCluster(int nodeCountPerAZ, Settings commonSettings) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java index 23d48b173a3db..253a8b2b14824 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java @@ -37,6 +37,7 @@ import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.action.search.SearchPhaseName; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.GroupShardsIterator; @@ -63,6 +64,7 @@ import java.util.Set; import java.util.function.Function; +import static org.opensearch.action.search.TransportSearchAction.SEARCH_REQUEST_STATS_ENABLED_KEY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; @@ -78,7 +80,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -@OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, minNumDataNodes = 2) public class SearchStatsIT extends ParameterizedOpenSearchIntegTestCase { public SearchStatsIT(Settings dynamicSettings) { @@ -126,6 +128,11 @@ public void testSimpleStats() throws Exception { assertThat(numNodes, greaterThanOrEqualTo(2)); final int shardsIdx1 = randomIntBetween(1, 10); // we make sure each node gets at least a single shard... final int shardsIdx2 = Math.max(numNodes - shardsIdx1, randomIntBetween(1, 10)); + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(SEARCH_REQUEST_STATS_ENABLED_KEY, true).build()) + .get(); assertThat(numNodes, lessThanOrEqualTo(shardsIdx1 + shardsIdx2)); assertAcked( prepareCreate("test1").setSettings( @@ -188,20 +195,40 @@ public void testSimpleStats() throws Exception { Set nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2"); int num = 0; + int numOfCoordinators = 0; + for (NodeStats stat : nodeStats.getNodes()) { Stats total = stat.getIndices().getSearch().getTotal(); + if (total.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.QUERY.getName()).getTimeInMillis() > 0) { + assertThat( + total.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.FETCH.getName()).getTimeInMillis(), + greaterThan(0L) + ); + assertEquals( + iters, + total.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.FETCH.getName()).getTotal() + ); + assertEquals( + iters, + total.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.EXPAND.getName()).getTotal() + ); + assertEquals( + iters, + total.getRequestStatsLongHolder().getRequestStatsHolder().get(SearchPhaseName.FETCH.getName()).getTotal() + ); + numOfCoordinators += 1; + } if (nodeIdsWithIndex.contains(stat.getNode().getId())) { assertThat(total.getQueryCount(), greaterThan(0L)); assertThat(total.getQueryTimeInMillis(), greaterThan(0L)); num++; } else { - assertThat(total.getQueryCount(), equalTo(0L)); + assertThat(total.getQueryCount(), greaterThanOrEqualTo(0L)); assertThat(total.getQueryTimeInMillis(), equalTo(0L)); } } - + assertThat(numOfCoordinators, greaterThan(0)); assertThat(num, greaterThan(0)); - } private Set nodeIdsWithIndex(String... indices) { diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index ee8aa10577956..1c0a1280ad550 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -65,6 +65,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; @@ -107,7 +108,6 @@ abstract class AbstractSearchAsyncAction exten private final AtomicInteger skippedOps = new AtomicInteger(); private final TransportSearchAction.SearchTimeProvider timeProvider; private final SearchResponse.Clusters clusters; - protected final GroupShardsIterator toSkipShardsIts; protected final GroupShardsIterator shardsIts; private final int expectedTotalOps; @@ -116,8 +116,12 @@ abstract class AbstractSearchAsyncAction exten private final Map pendingExecutionsPerNode = new ConcurrentHashMap<>(); private final boolean throttleConcurrentRequests; + private SearchPhase currentPhase; + private final List releasables = new ArrayList<>(); + private Optional searchRequestOperationsListener; + AbstractSearchAsyncAction( String name, Logger logger, @@ -135,7 +139,8 @@ abstract class AbstractSearchAsyncAction exten SearchTask task, SearchPhaseResults resultConsumer, int maxConcurrentRequestsPerNode, - SearchResponse.Clusters clusters + SearchResponse.Clusters clusters, + SearchRequestOperationsListener searchRequestOperationsListener ) { super(name); final List toSkipIterators = new ArrayList<>(); @@ -171,6 +176,7 @@ abstract class AbstractSearchAsyncAction exten this.indexRoutings = indexRoutings; this.results = resultConsumer; this.clusters = clusters; + this.searchRequestOperationsListener = Optional.ofNullable(searchRequestOperationsListener); } @Override @@ -371,6 +377,7 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha : OpenSearchException.guessRootCauses(shardSearchFailures[0].getCause())[0]; logger.debug(() -> new ParameterizedMessage("All shards failed for phase: [{}]", getName()), cause); onPhaseFailure(currentPhase, "all shards failed", cause); + } else { Boolean allowPartialResults = request.allowPartialSearchResults(); assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults"; @@ -419,13 +426,24 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha clusterState.version() ); } + onPhaseEnd(); executePhase(nextPhase); } } + private void onPhaseEnd() { + this.searchRequestOperationsListener.ifPresent(searchRequestOperations -> { searchRequestOperations.onPhaseEnd(this); }); + } + + private void onPhaseStart(SearchPhase phase) { + setCurrentPhase(phase); + this.searchRequestOperationsListener.ifPresent(searchRequestOperations -> { searchRequestOperations.onPhaseStart(this); }); + } + private void executePhase(SearchPhase phase) { try { - phase.run(); + onPhaseStart(phase); + phase.recordAndRun(); } catch (Exception e) { if (logger.isDebugEnabled()) { logger.debug(new ParameterizedMessage("Failed to execute [{}] while moving to [{}] phase", request, phase.getName()), e); @@ -603,6 +621,14 @@ private void successfulShardExecution(SearchShardIterator shardsIt) { } } + public SearchPhase getCurrentPhase() { + return currentPhase; + } + + private void setCurrentPhase(SearchPhase phase) { + currentPhase = phase; + } + @Override public final int getNumShards() { return results.getNumShards(); @@ -670,10 +696,13 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At } listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId)); } + onPhaseEnd(); + setCurrentPhase(null); } @Override public final void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) { + this.searchRequestOperationsListener.ifPresent(searchRequestOperations -> searchRequestOperations.onPhaseFailure(this)); raisePhaseFailure(new SearchPhaseExecutionException(phase.getName(), msg, cause, buildShardFailures())); } diff --git a/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java index 6c3ee652de2de..ae481736ad0aa 100644 --- a/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/CanMatchPreFilterSearchPhase.java @@ -90,7 +90,8 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction, SearchPhase> phaseFactory, - SearchResponse.Clusters clusters + SearchResponse.Clusters clusters, + SearchRequestOperationsListener searchRequestOperationsListener ) { // We set max concurrent shard requests to the number of shards so no throttling happens for can_match requests super( @@ -110,7 +111,8 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction(shardsIts.size()), request.getMaxConcurrentShardRequests(), - clusters + clusters, + searchRequestOperationsListener ); this.queryPhaseResultConsumer = queryPhaseResultConsumer; this.searchPhaseController = searchPhaseController; diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhase.java b/server/src/main/java/org/opensearch/action/search/SearchPhase.java index 50b0cd8e01c1d..1c7b3c1f1563c 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhase.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhase.java @@ -42,13 +42,23 @@ * * @opensearch.internal */ -abstract class SearchPhase implements CheckedRunnable { +public abstract class SearchPhase implements CheckedRunnable { private final String name; + private long startTimeInNanos; protected SearchPhase(String name) { this.name = Objects.requireNonNull(name, "name must not be null"); } + public long getStartTimeInNanos() { + return startTimeInNanos; + } + + public void recordAndRun() throws IOException { + this.startTimeInNanos = System.nanoTime(); + run(); + } + /** * Returns the phases name. */ diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java index 4ffd5521793f6..45d39a6f85ea2 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java @@ -73,6 +73,8 @@ public interface SearchPhaseContext extends Executor { */ SearchRequest getRequest(); + SearchPhase getCurrentPhase(); + /** * Builds and sends the final search response back to the user. * diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java index b6f842cf2cce1..4c0fe3ac06326 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java @@ -13,6 +13,7 @@ * @opensearch.internal */ public enum SearchPhaseName { + DFS_PRE_QUERY("dfs_pre_query"), QUERY("query"), FETCH("fetch"), DFS_QUERY("dfs_query"), diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java index f75ab2554e693..ca5ad087d3089 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java @@ -81,10 +81,11 @@ class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction listeners; + private final Logger logger; + + public CompositeListener(List listeners, Logger logger) { + this.listeners = listeners; + this.logger = logger; + } + + @Override + public void onPhaseStart(SearchPhaseContext context) { + for (SearchRequestOperationsListener listener : listeners) { + try { + listener.onPhaseStart(context); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("onPhaseStart listener [{}] failed", listener), e); + } + } + } + + @Override + public void onPhaseEnd(SearchPhaseContext context) { + for (SearchRequestOperationsListener listener : listeners) { + try { + listener.onPhaseEnd(context); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("onPhaseEnd listener [{}] failed", listener), e); + } + } + } + + @Override + public void onPhaseFailure(SearchPhaseContext context) { + for (SearchRequestOperationsListener listener : listeners) { + try { + listener.onPhaseFailure(context); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("onPhaseFailure listener [{}] failed", listener), e); + } + } + } + } +} diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java new file mode 100644 index 0000000000000..ad299c11b987d --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.search; + +import org.opensearch.common.inject.Inject; +import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.common.metrics.MeanMetric; + +import java.util.EnumMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Request level search stats to track coordinator level node search latencies + * + * @opensearch.internal + */ +public final class SearchRequestStats implements SearchRequestOperationsListener { + Map phaseStatsMap = new EnumMap<>(SearchPhaseName.class); + + @Inject + public SearchRequestStats() { + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + phaseStatsMap.put(searchPhaseName, new StatsHolder()); + } + } + + public long getPhaseCurrent(SearchPhaseName searchPhaseName) { + return phaseStatsMap.get(searchPhaseName).current.count(); + } + + public long getPhaseTotal(SearchPhaseName searchPhaseName) { + return phaseStatsMap.get(searchPhaseName).total.count(); + } + + public long getPhaseMetric(SearchPhaseName searchPhaseName) { + return phaseStatsMap.get(searchPhaseName).timing.sum(); + } + + @Override + public void onPhaseStart(SearchPhaseContext context) { + phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.inc(); + } + + @Override + public void onPhaseEnd(SearchPhaseContext context) { + StatsHolder phaseStats = phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()); + phaseStats.current.dec(); + phaseStats.total.inc(); + phaseStats.timing.inc(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - context.getCurrentPhase().getStartTimeInNanos())); + } + + @Override + public void onPhaseFailure(SearchPhaseContext context) { + phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.dec(); + } + + /** + * Holder of statistics values + * + * @opensearch.internal + */ + + public static final class StatsHolder { + CounterMetric current = new CounterMetric(); + CounterMetric total = new CounterMetric(); + MeanMetric timing = new MeanMetric(); + } +} diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 25ec0fc57d19f..cff1005beff27 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -67,6 +67,7 @@ import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.indices.breaker.CircuitBreakerService; @@ -145,6 +146,14 @@ public class TransportSearchAction extends HandledTransportAction SEARCH_REQUEST_STATS_ENABLED = Setting.boolSetting( + SEARCH_REQUEST_STATS_ENABLED_KEY, + false, + Property.Dynamic, + Property.NodeScope + ); + private final NodeClient client; private final ThreadPool threadPool; private final ClusterService clusterService; @@ -157,6 +166,10 @@ public class TransportSearchAction extends HandledTransportAction) SearchRequest::new); this.client = client; @@ -185,6 +199,13 @@ public TransportSearchAction( this.indexNameExpressionResolver = indexNameExpressionResolver; this.namedWriteableRegistry = namedWriteableRegistry; this.searchPipelineService = searchPipelineService; + this.isRequestStatsEnabled = clusterService.getClusterSettings().get(SEARCH_REQUEST_STATS_ENABLED); + clusterService.getClusterSettings().addSettingsUpdateConsumer(SEARCH_REQUEST_STATS_ENABLED, this::setIsRequestStatsEnabled); + this.searchRequestStats = searchRequestStats; + } + + private void setIsRequestStatsEnabled(boolean isRequestStatsEnabled) { + this.isRequestStatsEnabled = isRequestStatsEnabled; } private Map buildPerIndexAliasFilter( @@ -311,6 +332,13 @@ public void executeRequest( SinglePhaseSearchAction phaseSearchAction, ActionListener listener ) { + final List searchListenersList = createSearchListenerList(); + final SearchRequestOperationsListener searchRequestOperationsListener; + if (!CollectionUtils.isEmpty(searchListenersList)) { + searchRequestOperationsListener = new SearchRequestOperationsListener.CompositeListener(searchListenersList, logger); + } else { + searchRequestOperationsListener = null; + } executeRequest(task, searchRequest, new SearchAsyncActionProvider() { @Override public AbstractSearchAsyncAction asyncSearchAction( @@ -346,7 +374,8 @@ public AbstractSearchAsyncAction asyncSearchAction( task, new ArraySearchPhaseResults<>(shardsIts.size()), searchRequest.getMaxConcurrentShardRequests(), - clusters + clusters, + searchRequestOperationsListener ) { @Override protected void executePhaseOnShard( @@ -916,9 +945,7 @@ private void executeSearch( @Nullable SearchContextId searchContext, SearchAsyncActionProvider searchAsyncActionProvider ) { - clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ); - // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead // of just for the _search api @@ -968,11 +995,8 @@ private void executeSearch( indexRoutings = routingMap; } final GroupShardsIterator shardIterators = mergeShardsIterators(localShardIterators, remoteShardIterators); - failIfOverShardCountLimit(clusterService, shardIterators.size()); - Map concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState); - // optimize search type for cases where there is only one shard group to search on if (shardIterators.size() == 1) { // if we only have one group, then we always want Q_T_F, no need for DFS, and no need to do THEN since we hit one shard @@ -1107,6 +1131,14 @@ AbstractSearchAsyncAction asyncSearchAction( ); } + private List createSearchListenerList() { + final List searchListenersList = new ArrayList<>(); + if (isRequestStatsEnabled) { + searchListenersList.add(searchRequestStats); + } + return searchListenersList; + } + private AbstractSearchAsyncAction searchAsyncAction( SearchTask task, SearchRequest searchRequest, @@ -1123,6 +1155,13 @@ private AbstractSearchAsyncAction searchAsyncAction ThreadPool threadPool, SearchResponse.Clusters clusters ) { + final List searchListenersList = createSearchListenerList(); + final SearchRequestOperationsListener searchRequestOperationsListener; + if (!CollectionUtils.isEmpty(searchListenersList)) { + searchRequestOperationsListener = new SearchRequestOperationsListener.CompositeListener(searchListenersList, logger); + } else { + searchRequestOperationsListener = null; + } if (preFilter) { return new CanMatchPreFilterSearchPhase( logger, @@ -1162,7 +1201,8 @@ public void run() { } }; }, - clusters + clusters, + searchRequestOperationsListener ); } else { final QueryPhaseResultConsumer queryResultConsumer = searchPhaseController.newSearchPhaseResults( @@ -1192,7 +1232,8 @@ public void run() { timeProvider, clusterState, task, - clusters + clusters, + searchRequestOperationsListener ); break; case QUERY_THEN_FETCH: @@ -1212,7 +1253,8 @@ public void run() { timeProvider, clusterState, task, - clusters + clusters, + searchRequestOperationsListener ); break; default: diff --git a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java index 5c48c1f772ff0..cb181840406a5 100644 --- a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java +++ b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java @@ -62,4 +62,5 @@ public void dec(long n) { public long count() { return counter.sum(); } + } diff --git a/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java b/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java index 33f12c8cb42d3..359facdce633b 100644 --- a/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java +++ b/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java @@ -79,4 +79,5 @@ public void clear() { counter.reset(); sum.reset(); } + } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 74224d66400da..22e65b2c04668 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -373,6 +373,7 @@ public void apply(Settings value, Settings current, Settings previous) { SearchService.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS, TransportSearchAction.SHARD_COUNT_LIMIT_SETTING, TransportSearchAction.SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING, + TransportSearchAction.SEARCH_REQUEST_STATS_ENABLED, RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE, SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER, RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING, diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java index 76b216304b3b7..14aaf7e58a59c 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java @@ -33,6 +33,8 @@ package org.opensearch.index.search.stats; import org.opensearch.Version; +import org.opensearch.action.search.SearchPhaseName; +import org.opensearch.action.search.SearchRequestStats; import org.opensearch.common.Nullable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; @@ -58,10 +60,79 @@ public class SearchStats implements Writeable, ToXContentFragment { /** - * Statistics for search + * Holds statistic values for a particular phase. * * @opensearch.internal */ + public static class PhaseStatsLongHolder implements Writeable { + + long current; + long total; + long timeInMillis; + + public long getCurrent() { + return current; + } + + public long getTotal() { + return total; + } + + public long getTimeInMillis() { + return timeInMillis; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(current); + out.writeVLong(total); + out.writeVLong(timeInMillis); + } + + PhaseStatsLongHolder() { + this(0, 0, 0); + } + + PhaseStatsLongHolder(long current, long total, long timeInMillis) { + this.current = current; + this.total = total; + this.timeInMillis = timeInMillis; + } + + PhaseStatsLongHolder(StreamInput in) throws IOException { + this.current = in.readVLong(); + this.total = in.readVLong(); + this.timeInMillis = in.readVLong(); + } + + } + + /** + * Holds requests stats for different phases. + * + * @opensearch.internal + */ + public static class RequestStatsLongHolder { + + Map requestStatsHolder = new HashMap<>(); + + public Map getRequestStatsHolder() { + return requestStatsHolder; + } + + RequestStatsLongHolder() { + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + requestStatsHolder.put(searchPhaseName.getName(), new PhaseStatsLongHolder()); + } + } + } + + /** + * Holder of statistics values + * + * @opensearch.internal + */ + public static class Stats implements Writeable, ToXContentFragment { private long queryCount; @@ -89,6 +160,13 @@ public static class Stats implements Writeable, ToXContentFragment { private long pitTimeInMillis; private long pitCurrent; + @Nullable + private RequestStatsLongHolder requestStatsLongHolder; + + public RequestStatsLongHolder getRequestStatsLongHolder() { + return requestStatsLongHolder; + } + private Stats() { // for internal use, initializes all counts to 0 } @@ -114,6 +192,7 @@ public Stats( long suggestTimeInMillis, long suggestCurrent ) { + this.requestStatsLongHolder = new RequestStatsLongHolder(); this.queryCount = queryCount; this.queryTimeInMillis = queryTimeInMillis; this.queryCurrent = queryCurrent; @@ -163,6 +242,10 @@ private Stats(StreamInput in) throws IOException { pitCurrent = in.readVLong(); } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.requestStatsLongHolder = new RequestStatsLongHolder(); + requestStatsLongHolder.requestStatsHolder = in.readMap(StreamInput::readString, PhaseStatsLongHolder::new); + } if (in.getVersion().onOrAfter(Version.V_2_10_0)) { concurrentQueryCount = in.readVLong(); concurrentQueryTimeInMillis = in.readVLong(); @@ -354,6 +437,17 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(pitCurrent); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (requestStatsLongHolder == null) { + requestStatsLongHolder = new RequestStatsLongHolder(); + } + out.writeMap( + requestStatsLongHolder.getRequestStatsHolder(), + StreamOutput::writeString, + (stream, stats) -> stats.writeTo(stream) + ); + } + if (out.getVersion().onOrAfter(Version.V_2_10_0)) { out.writeVLong(concurrentQueryCount); out.writeVLong(concurrentQueryTimeInMillis); @@ -391,6 +485,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.humanReadableField(Fields.SUGGEST_TIME_IN_MILLIS, Fields.SUGGEST_TIME, getSuggestTime()); builder.field(Fields.SUGGEST_CURRENT, suggestCurrent); + if (requestStatsLongHolder != null) { + builder.startObject(Fields.REQUEST); + + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + PhaseStatsLongHolder statsLongHolder = requestStatsLongHolder.requestStatsHolder.get(searchPhaseName.getName()); + if (statsLongHolder == null) { + continue; + } + builder.startObject(searchPhaseName.getName()); + builder.humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(statsLongHolder.timeInMillis)); + builder.field(Fields.CURRENT, statsLongHolder.current); + builder.field(Fields.TOTAL, statsLongHolder.total); + builder.endObject(); + } + builder.endObject(); + } return builder; } } @@ -405,6 +515,24 @@ public SearchStats() { totalStats = new Stats(); } + // Set the different Request Stats fields in here + public void setSearchRequestStats(SearchRequestStats searchRequestStats) { + if (totalStats.requestStatsLongHolder == null) { + totalStats.requestStatsLongHolder = new RequestStatsLongHolder(); + } + + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + totalStats.requestStatsLongHolder.requestStatsHolder.put( + searchPhaseName.getName(), + new PhaseStatsLongHolder( + searchRequestStats.getPhaseCurrent(searchPhaseName), + searchRequestStats.getPhaseTotal(searchPhaseName), + searchRequestStats.getPhaseMetric(searchPhaseName) + ) + ); + } + } + public SearchStats(Stats totalStats, long openContexts, @Nullable Map groupStats) { this.totalStats = totalStats; this.openContexts = openContexts; @@ -520,6 +648,12 @@ static final class Fields { static final String SUGGEST_TIME = "suggest_time"; static final String SUGGEST_TIME_IN_MILLIS = "suggest_time_in_millis"; static final String SUGGEST_CURRENT = "suggest_current"; + static final String REQUEST = "request"; + static final String TIME_IN_MILLIS = "time_in_millis"; + static final String TIME = "time"; + static final String CURRENT = "current"; + static final String TOTAL = "total"; + } @Override diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 2ed3cb8d9e8ea..cf64b886ed523 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -47,6 +47,7 @@ import org.opensearch.action.admin.indices.stats.CommonStatsFlags.Flag; import org.opensearch.action.admin.indices.stats.IndexShardStats; import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.search.SearchRequestStats; import org.opensearch.action.search.SearchType; import org.opensearch.client.Client; import org.opensearch.cluster.ClusterState; @@ -333,6 +334,8 @@ public class IndicesService extends AbstractLifecycleComponent private volatile TimeValue clusterRemoteTranslogBufferInterval; private final FileCacheCleaner fileCacheCleaner; + private final SearchRequestStats searchRequestStats; + @Override protected void doStart() { // Start thread that will manage cleaning the field data cache periodically @@ -363,6 +366,7 @@ public IndicesService( IndexStorePlugin.DirectoryFactory remoteDirectoryFactory, Supplier repositoriesServiceSupplier, FileCacheCleaner fileCacheCleaner, + SearchRequestStats searchRequestStats, @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory ) { this.settings = settings; @@ -453,6 +457,7 @@ protected void closeInternal() { clusterService.getClusterSettings().addSettingsUpdateConsumer(ALLOW_EXPENSIVE_QUERIES, this::setAllowExpensiveQueries); this.remoteDirectoryFactory = remoteDirectoryFactory; this.translogFactorySupplier = getTranslogFactorySupplier(repositoriesServiceSupplier, threadPool, remoteStoreStatsTrackerFactory); + this.searchRequestStats = searchRequestStats; this.clusterDefaultRefreshInterval = CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.get(clusterService.getSettings()); clusterService.getClusterSettings() .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); @@ -576,7 +581,7 @@ public NodeIndicesStats stats(CommonStatsFlags flags) { } } - return new NodeIndicesStats(commonStats, statsByShard(this, flags)); + return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats); } Map> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) { diff --git a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java index cc3d8193dfa6b..8a7aaba2726f4 100644 --- a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java +++ b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java @@ -35,6 +35,7 @@ import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndexShardStats; import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.search.SearchRequestStats; import org.opensearch.common.Nullable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -71,7 +72,6 @@ * @opensearch.internal */ public class NodeIndicesStats implements Writeable, ToXContentFragment { - private CommonStats stats; private Map> statsByShard; @@ -92,7 +92,7 @@ public NodeIndicesStats(StreamInput in) throws IOException { } } - public NodeIndicesStats(CommonStats oldStats, Map> statsByShard) { + public NodeIndicesStats(CommonStats oldStats, Map> statsByShard, SearchRequestStats searchRequestStats) { // this.stats = stats; this.statsByShard = statsByShard; @@ -105,6 +105,9 @@ public NodeIndicesStats(CommonStats oldStats, Map> } } } + if (this.stats.search != null) { + this.stats.search.setSearchRequestStats(searchRequestStats); + } } @Nullable diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 1f8f17f8e8d91..90fb339951d62 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -46,6 +46,7 @@ import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; import org.opensearch.action.search.SearchPhaseController; +import org.opensearch.action.search.SearchRequestStats; import org.opensearch.action.search.SearchTransportService; import org.opensearch.action.support.TransportAction; import org.opensearch.action.update.UpdateHelper; @@ -761,6 +762,8 @@ protected Node( threadPool ); + final SearchRequestStats searchRequestStats = new SearchRequestStats(); + remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); final IndicesService indicesService = new IndicesService( settings, @@ -786,6 +789,7 @@ protected Node( remoteDirectoryFactory, repositoriesServiceReference::get, fileCacheCleaner, + searchRequestStats, remoteStoreStatsTrackerFactory ); @@ -1199,6 +1203,7 @@ protected Node( b.bind(SystemIndices.class).toInstance(systemIndices); b.bind(IdentityService.class).toInstance(identityService); b.bind(Tracer.class).toInstance(tracer); + b.bind(SearchRequestStats.class).toInstance(searchRequestStats); b.bind(RemoteClusterStateService.class).toProvider(() -> remoteClusterStateService); b.bind(PersistedStateRegistry.class).toInstance(persistedStateRegistry); }); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index 672ebea8d01f9..e3f16463a5328 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -34,6 +34,7 @@ import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.action.search.SearchRequestStats; import org.opensearch.cluster.coordination.PendingClusterStateStats; import org.opensearch.cluster.coordination.PublishClusterStateStats; import org.opensearch.cluster.node.DiscoveryNode; @@ -801,8 +802,7 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) { private static NodeIndicesStats getNodeIndicesStats(boolean remoteStoreStats) { NodeIndicesStats indicesStats = null; if (remoteStoreStats) { - indicesStats = new NodeIndicesStats(new CommonStats(CommonStatsFlags.ALL), new HashMap<>()); - + indicesStats = new NodeIndicesStats(new CommonStats(CommonStatsFlags.ALL), new HashMap<>(), new SearchRequestStats()); RemoteSegmentStats remoteSegmentStats = indicesStats.getSegments().getRemoteSegmentStats(); remoteSegmentStats.addUploadBytesStarted(10L); remoteSegmentStats.addUploadBytesSucceeded(10L); diff --git a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java index 705479ec21fc1..f628bb3201452 100644 --- a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java @@ -38,8 +38,12 @@ import org.opensearch.cluster.routing.GroupShardsIterator; import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.util.concurrent.AtomicArray; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.set.Sets; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.query.MatchAllQueryBuilder; @@ -51,7 +55,10 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.test.InternalAggregationTestCase; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; import org.junit.After; import org.junit.Before; @@ -65,6 +72,7 @@ import java.util.UUID; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -77,18 +85,21 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; public class AbstractSearchAsyncActionTests extends OpenSearchTestCase { private final List> resolvedNodes = new ArrayList<>(); private final Set releasedContexts = new CopyOnWriteArraySet<>(); private ExecutorService executor; + ThreadPool threadPool; @Before @Override public void setUp() throws Exception { super.setUp(); executor = Executors.newFixedThreadPool(1); + threadPool = new TestThreadPool(getClass().getName()); } @After @@ -97,6 +108,7 @@ public void tearDown() throws Exception { super.tearDown(); executor.shutdown(); assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + ThreadPool.terminate(threadPool, 5, TimeUnit.SECONDS); } private AbstractSearchAsyncAction createAction( @@ -126,6 +138,7 @@ private AbstractSearchAsyncAction createAction( final AtomicLong expected, final SearchShardIterator... shards ) { + final Runnable runnable; final TransportSearchAction.SearchTimeProvider timeProvider; if (controlled) { @@ -161,7 +174,8 @@ private AbstractSearchAsyncAction createAction( null, results, request.getMaxConcurrentShardRequests(), - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { @Override protected SearchPhase getNextPhase(final SearchPhaseResults results, SearchPhaseContext context) { @@ -313,6 +327,53 @@ public void testSendSearchResponseDisallowPartialFailures() { assertEquals(requestIds, releasedContexts); } + public void testOnPhaseFailureAndVerifyListeners() { + SearchRequestStats testListener = new SearchRequestStats(); + + final List requestOperationListeners = new ArrayList<>(List.of(testListener)); + SearchQueryThenFetchAsyncAction action = createSearchQueryThenFetchAsyncAction(requestOperationListeners); + action.start(); + assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); + action.onPhaseFailure(new SearchPhase("test") { + @Override + public void run() { + + } + }, "message", null); + assertEquals(0, testListener.getPhaseCurrent(action.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseTotal(action.getSearchPhaseName())); + + SearchDfsQueryThenFetchAsyncAction searchDfsQueryThenFetchAsyncAction = createSearchDfsQueryThenFetchAsyncAction( + requestOperationListeners + ); + searchDfsQueryThenFetchAsyncAction.start(); + assertEquals(1, testListener.getPhaseCurrent(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); + searchDfsQueryThenFetchAsyncAction.onPhaseFailure(new SearchPhase("test") { + @Override + public void run() { + + } + }, "message", null); + assertEquals(0, testListener.getPhaseCurrent(action.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseTotal(action.getSearchPhaseName())); + + FetchSearchPhase fetchPhase = createFetchSearchPhase(); + ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); + SearchShardIterator searchShardIterator = new SearchShardIterator(null, shardId, Collections.emptyList(), OriginalIndices.NONE); + searchShardIterator.resetAndSkip(); + action.skipShard(searchShardIterator); + action.executeNextPhase(action, fetchPhase); + assertEquals(1, testListener.getPhaseCurrent(fetchPhase.getSearchPhaseName())); + action.onPhaseFailure(new SearchPhase("test") { + @Override + public void run() { + + } + }, "message", null); + assertEquals(0, testListener.getPhaseCurrent(fetchPhase.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseTotal(fetchPhase.getSearchPhaseName())); + } + public void testOnPhaseFailure() { SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false); AtomicReference exception = new AtomicReference<>(); @@ -321,6 +382,7 @@ public void testOnPhaseFailure() { List> nodeLookups = new ArrayList<>(); ArraySearchPhaseResults phaseResults = phaseResults(requestIds, nodeLookups, 0); AbstractSearchAsyncAction action = createAction(searchRequest, phaseResults, listener, false, new AtomicLong()); + action.onPhaseFailure(new SearchPhase("test") { @Override public void run() { @@ -528,6 +590,215 @@ public void onFailure(Exception e) { assertThat(searchResponse.getSuccessfulShards(), equalTo(shards.length)); } + public void testOnPhaseListenersWithQueryAndThenFetchType() throws InterruptedException { + SearchRequestStats testListener = new SearchRequestStats(); + final List requestOperationListeners = new ArrayList<>(List.of(testListener)); + + long delay = (randomIntBetween(1, 5)); + delay = delay * 10; + + SearchQueryThenFetchAsyncAction action = createSearchQueryThenFetchAsyncAction(requestOperationListeners); + action.start(); + + // Verify queryPhase current metric + assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); + TimeUnit.MILLISECONDS.sleep(delay); + + FetchSearchPhase fetchPhase = createFetchSearchPhase(); + ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); + SearchShardIterator searchShardIterator = new SearchShardIterator(null, shardId, Collections.emptyList(), OriginalIndices.NONE); + searchShardIterator.resetAndSkip(); + action.skipShard(searchShardIterator); + action.executeNextPhase(action, fetchPhase); + + // Verify queryPhase total, current and latency metrics + assertEquals(0, testListener.getPhaseCurrent(action.getSearchPhaseName())); + assertThat(testListener.getPhaseMetric(action.getSearchPhaseName()), greaterThanOrEqualTo(delay)); + assertEquals(1, testListener.getPhaseTotal(action.getSearchPhaseName())); + + // Verify fetchPhase current metric + assertEquals(1, testListener.getPhaseCurrent(fetchPhase.getSearchPhaseName())); + TimeUnit.MILLISECONDS.sleep(delay); + + ExpandSearchPhase expandPhase = createExpandSearchPhase(); + action.executeNextPhase(fetchPhase, expandPhase); + TimeUnit.MILLISECONDS.sleep(delay); + + // Verify fetchPhase total, current and latency metrics + assertThat(testListener.getPhaseMetric(fetchPhase.getSearchPhaseName()), greaterThanOrEqualTo(delay)); + assertEquals(1, testListener.getPhaseTotal(fetchPhase.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseCurrent(fetchPhase.getSearchPhaseName())); + + assertEquals(1, testListener.getPhaseCurrent(expandPhase.getSearchPhaseName())); + + action.executeNextPhase(expandPhase, fetchPhase); + + action.sendSearchResponse(mock(InternalSearchResponse.class), mock(String.valueOf(QuerySearchResult.class))); + assertThat(testListener.getPhaseMetric(expandPhase.getSearchPhaseName()), greaterThanOrEqualTo(delay)); + assertEquals(1, testListener.getPhaseTotal(expandPhase.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseCurrent(expandPhase.getSearchPhaseName())); + } + + public void testOnPhaseListenersWithDfsType() throws InterruptedException { + SearchRequestStats testListener = new SearchRequestStats(); + final List requestOperationListeners = new ArrayList<>(List.of(testListener)); + + SearchDfsQueryThenFetchAsyncAction searchDfsQueryThenFetchAsyncAction = createSearchDfsQueryThenFetchAsyncAction( + requestOperationListeners + ); + long delay = (randomIntBetween(1, 5)); + + FetchSearchPhase fetchPhase = createFetchSearchPhase(); + searchDfsQueryThenFetchAsyncAction.start(); + assertEquals(1, testListener.getPhaseCurrent(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); + TimeUnit.MILLISECONDS.sleep(delay); + ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); + SearchShardIterator searchShardIterator = new SearchShardIterator(null, shardId, Collections.emptyList(), OriginalIndices.NONE); + searchShardIterator.resetAndSkip(); + + searchDfsQueryThenFetchAsyncAction.skipShard(searchShardIterator); + searchDfsQueryThenFetchAsyncAction.executeNextPhase(searchDfsQueryThenFetchAsyncAction, fetchPhase); + + assertThat(testListener.getPhaseMetric(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName()), greaterThanOrEqualTo(delay)); + assertEquals(1, testListener.getPhaseTotal(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseCurrent(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); + } + + private SearchDfsQueryThenFetchAsyncAction createSearchDfsQueryThenFetchAsyncAction( + List searchRequestOperationsListeners + ) { + SearchPhaseController controller = new SearchPhaseController( + writableRegistry(), + r -> InternalAggregationTestCase.emptyReduceContextBuilder() + ); + SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); + SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); + Executor executor = OpenSearchExecutors.newDirectExecutorService(); + SearchShardIterator shards = new SearchShardIterator(null, null, Collections.emptyList(), null); + GroupShardsIterator shardsIter = new GroupShardsIterator<>(List.of(shards)); + QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( + searchRequest, + executor, + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + controller, + task.getProgressListener(), + writableRegistry(), + shardsIter.size(), + exc -> {} + ); + AtomicReference exception = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set); + + return new SearchDfsQueryThenFetchAsyncAction( + logger, + null, + null, + null, + null, + null, + null, + executor, + resultConsumer, + searchRequest, + listener, + shardsIter, + null, + null, + task, + SearchResponse.Clusters.EMPTY, + new SearchRequestOperationsListener.CompositeListener(searchRequestOperationsListeners, logger) + ); + } + + private SearchQueryThenFetchAsyncAction createSearchQueryThenFetchAsyncAction( + List searchRequestOperationsListeners + ) { + SearchPhaseController controller = new SearchPhaseController( + writableRegistry(), + r -> InternalAggregationTestCase.emptyReduceContextBuilder() + ); + SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); + SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); + Executor executor = OpenSearchExecutors.newDirectExecutorService(); + SearchShardIterator shards = new SearchShardIterator(null, null, Collections.emptyList(), null); + GroupShardsIterator shardsIter = new GroupShardsIterator<>(List.of(shards)); + QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( + searchRequest, + executor, + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + controller, + task.getProgressListener(), + writableRegistry(), + shardsIter.size(), + exc -> {} + ); + AtomicReference exception = new AtomicReference<>(); + ActionListener listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set); + return new SearchQueryThenFetchAsyncAction( + logger, + null, + null, + null, + null, + null, + null, + executor, + resultConsumer, + searchRequest, + listener, + shardsIter, + null, + null, + task, + SearchResponse.Clusters.EMPTY, + new SearchRequestOperationsListener.CompositeListener(searchRequestOperationsListeners, logger) + ) { + @Override + ShardSearchFailure[] buildShardFailures() { + return ShardSearchFailure.EMPTY_ARRAY; + } + + @Override + public void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray queryResults) { + start(); + } + }; + } + + private FetchSearchPhase createFetchSearchPhase() { + SearchPhaseController controller = new SearchPhaseController( + writableRegistry(), + r -> InternalAggregationTestCase.emptyReduceContextBuilder() + ); + MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); + QueryPhaseResultConsumer results = controller.newSearchPhaseResults( + OpenSearchExecutors.newDirectExecutorService(), + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + SearchProgressListener.NOOP, + mockSearchPhaseContext.getRequest(), + 1, + exc -> {} + ); + return new FetchSearchPhase( + results, + controller, + null, + mockSearchPhaseContext, + (searchResponse, scrollId) -> new SearchPhase("test") { + @Override + public void run() { + mockSearchPhaseContext.sendSearchResponse(searchResponse, null); + } + } + ); + } + + private ExpandSearchPhase createExpandSearchPhase() { + MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); + InternalSearchResponse internalSearchResponse = new InternalSearchResponse(null, null, null, null, false, null, 1); + return new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, null); + } + private static final class PhaseResult extends SearchPhaseResult { PhaseResult(ShardSearchContextId contextId) { this.contextId = contextId; diff --git a/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java index 45f00a8418d5c..43029fe57d5dd 100644 --- a/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -136,7 +136,8 @@ public void run() throws IOException { latch.countDown(); } }, - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ); canMatchPhase.start(); @@ -227,7 +228,8 @@ public void run() throws IOException { latch.countDown(); } }, - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ); canMatchPhase.start(); @@ -317,7 +319,8 @@ public void sendCanMatch( null, new ArraySearchPhaseResults<>(iter.size()), randomIntBetween(1, 32), - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { @Override @@ -344,7 +347,8 @@ protected void executePhaseOnShard( } } }, - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ); canMatchPhase.start(); @@ -428,7 +432,8 @@ public void run() { latch.countDown(); } }, - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ); canMatchPhase.start(); @@ -527,7 +532,8 @@ public void run() { latch.countDown(); } }, - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ); canMatchPhase.start(); diff --git a/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java index e078b4a467e91..b5e1050b968ee 100644 --- a/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java +++ b/server/src/test/java/org/opensearch/action/search/MockSearchPhaseContext.java @@ -99,6 +99,11 @@ public SearchRequest getRequest() { return searchRequest; } + @Override + public SearchPhase getCurrentPhase() { + return null; + } + @Override public void sendSearchResponse(InternalSearchResponse internalSearchResponse, AtomicArray queryResults) { String scrollId = getRequest().scroll() != null ? TransportSearchHelper.buildScrollId(queryResults, Version.CURRENT) : null; diff --git a/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java index 830fa99f90bb9..4b94b6589c6c8 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchAsyncActionTests.java @@ -135,7 +135,8 @@ public void testSkipSearchShards() throws InterruptedException { null, new ArraySearchPhaseResults<>(shardsIter.size()), request.getMaxConcurrentShardRequests(), - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { @Override @@ -253,7 +254,8 @@ public void testLimitConcurrentShardRequests() throws InterruptedException { null, new ArraySearchPhaseResults<>(shardsIter.size()), request.getMaxConcurrentShardRequests(), - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { @Override @@ -370,7 +372,8 @@ public void sendFreeContext(Transport.Connection connection, ShardSearchContextI null, new ArraySearchPhaseResults<>(shardsIter.size()), request.getMaxConcurrentShardRequests(), - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { TestSearchResponse response = new TestSearchResponse(); @@ -492,7 +495,8 @@ public void sendFreeContext(Transport.Connection connection, ShardSearchContextI null, new ArraySearchPhaseResults<>(shardsIter.size()), request.getMaxConcurrentShardRequests(), - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { TestSearchResponse response = new TestSearchResponse(); @@ -605,9 +609,9 @@ public void testAllowPartialResults() throws InterruptedException { null, new ArraySearchPhaseResults<>(shardsIter.size()), request.getMaxConcurrentShardRequests(), - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { - @Override protected void executePhaseOnShard( SearchShardIterator shardIt, diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java index 0e2780c195cb8..6a22a7ea2b5e4 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -214,7 +214,8 @@ public void sendExecuteQuery( timeProvider, null, task, - SearchResponse.Clusters.EMPTY + SearchResponse.Clusters.EMPTY, + null ) { @Override protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { @@ -226,6 +227,7 @@ public void run() { }; } }; + action.start(); latch.await(); assertThat(successfulOps.get(), equalTo(numShards)); diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java new file mode 100644 index 0000000000000..ef880043e863c --- /dev/null +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.search; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SearchRequestOperationsListenerTests extends OpenSearchTestCase { + + public void testListenersAreExecuted() { + Map searchPhaseMap = new HashMap<>(); + + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + searchPhaseMap.put(searchPhaseName, new SearchRequestStats.StatsHolder()); + } + SearchRequestOperationsListener testListener = new SearchRequestOperationsListener() { + + @Override + public void onPhaseStart(SearchPhaseContext context) { + searchPhaseMap.get(context.getCurrentPhase().getSearchPhaseName()).current.inc(); + } + + @Override + public void onPhaseEnd(SearchPhaseContext context) { + searchPhaseMap.get(context.getCurrentPhase().getSearchPhaseName()).current.dec(); + searchPhaseMap.get(context.getCurrentPhase().getSearchPhaseName()).total.inc(); + } + + @Override + public void onPhaseFailure(SearchPhaseContext context) { + searchPhaseMap.get(context.getCurrentPhase().getSearchPhaseName()).current.dec(); + } + }; + + int totalListeners = randomIntBetween(1, 10); + final List requestOperationListeners = new ArrayList<>(); + for (int i = 0; i < totalListeners; i++) { + requestOperationListeners.add(testListener); + } + + SearchRequestOperationsListener compositeListener = new SearchRequestOperationsListener.CompositeListener( + requestOperationListeners, + logger + ); + + SearchPhaseContext ctx = mock(SearchPhaseContext.class); + SearchPhase searchPhase = mock(SearchPhase.class); + + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + when(ctx.getCurrentPhase()).thenReturn(searchPhase); + when(searchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); + compositeListener.onPhaseStart(ctx); + assertEquals(totalListeners, searchPhaseMap.get(searchPhaseName).current.count()); + } + } +} diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java new file mode 100644 index 0000000000000..f24147a8194b4 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java @@ -0,0 +1,150 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.search; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SearchRequestStatsTests extends OpenSearchTestCase { + public void testSearchRequestPhaseFailure() { + SearchRequestStats testRequestStats = new SearchRequestStats(); + SearchPhaseContext ctx = mock(SearchPhaseContext.class); + SearchPhase mockSearchPhase = mock(SearchPhase.class); + when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase); + + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); + testRequestStats.onPhaseStart(ctx); + assertEquals(1, testRequestStats.getPhaseCurrent(searchPhaseName)); + testRequestStats.onPhaseFailure(ctx); + assertEquals(0, testRequestStats.getPhaseCurrent(searchPhaseName)); + } + } + + public void testSearchRequestStats() { + SearchRequestStats testRequestStats = new SearchRequestStats(); + + SearchPhaseContext ctx = mock(SearchPhaseContext.class); + SearchPhase mockSearchPhase = mock(SearchPhase.class); + when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase); + + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); + long tookTimeInMillis = randomIntBetween(1, 10); + testRequestStats.onPhaseStart(ctx); + long startTime = System.nanoTime() - TimeUnit.MILLISECONDS.toNanos(tookTimeInMillis); + when(mockSearchPhase.getStartTimeInNanos()).thenReturn(startTime); + assertEquals(1, testRequestStats.getPhaseCurrent(searchPhaseName)); + testRequestStats.onPhaseEnd(ctx); + assertEquals(0, testRequestStats.getPhaseCurrent(searchPhaseName)); + assertEquals(1, testRequestStats.getPhaseTotal(searchPhaseName)); + assertThat(testRequestStats.getPhaseMetric(searchPhaseName), greaterThanOrEqualTo(tookTimeInMillis)); + } + } + + public void testSearchRequestStatsOnPhaseStartConcurrently() throws InterruptedException { + SearchRequestStats testRequestStats = new SearchRequestStats(); + int numTasks = randomIntBetween(5, 50); + Thread[] threads = new Thread[numTasks * SearchPhaseName.values().length]; + Phaser phaser = new Phaser(numTasks * SearchPhaseName.values().length + 1); + CountDownLatch countDownLatch = new CountDownLatch(numTasks * SearchPhaseName.values().length); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + SearchPhaseContext ctx = mock(SearchPhaseContext.class); + SearchPhase mockSearchPhase = mock(SearchPhase.class); + when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase); + when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); + for (int i = 0; i < numTasks; i++) { + threads[i] = new Thread(() -> { + phaser.arriveAndAwaitAdvance(); + testRequestStats.onPhaseStart(ctx); + countDownLatch.countDown(); + }); + threads[i].start(); + } + } + phaser.arriveAndAwaitAdvance(); + countDownLatch.await(); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + assertEquals(numTasks, testRequestStats.getPhaseCurrent(searchPhaseName)); + } + } + + public void testSearchRequestStatsOnPhaseEndConcurrently() throws InterruptedException { + SearchRequestStats testRequestStats = new SearchRequestStats(); + int numTasks = randomIntBetween(5, 50); + Thread[] threads = new Thread[numTasks * SearchPhaseName.values().length]; + Phaser phaser = new Phaser(numTasks * SearchPhaseName.values().length + 1); + CountDownLatch countDownLatch = new CountDownLatch(numTasks * SearchPhaseName.values().length); + Map searchPhaseNameLongMap = new HashMap<>(); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + SearchPhaseContext ctx = mock(SearchPhaseContext.class); + SearchPhase mockSearchPhase = mock(SearchPhase.class); + when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase); + when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); + long tookTimeInMillis = randomIntBetween(1, 10); + long startTime = System.nanoTime() - TimeUnit.MILLISECONDS.toNanos(tookTimeInMillis); + when(mockSearchPhase.getStartTimeInNanos()).thenReturn(startTime); + for (int i = 0; i < numTasks; i++) { + threads[i] = new Thread(() -> { + phaser.arriveAndAwaitAdvance(); + testRequestStats.onPhaseEnd(ctx); + countDownLatch.countDown(); + }); + threads[i].start(); + } + searchPhaseNameLongMap.put(searchPhaseName, tookTimeInMillis); + } + phaser.arriveAndAwaitAdvance(); + countDownLatch.await(); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + assertEquals(numTasks, testRequestStats.getPhaseTotal(searchPhaseName)); + assertThat( + testRequestStats.getPhaseMetric(searchPhaseName), + greaterThanOrEqualTo((searchPhaseNameLongMap.get(searchPhaseName) * numTasks)) + ); + } + } + + public void testSearchRequestStatsOnPhaseFailureConcurrently() throws InterruptedException { + SearchRequestStats testRequestStats = new SearchRequestStats(); + int numTasks = randomIntBetween(5, 50); + Thread[] threads = new Thread[numTasks * SearchPhaseName.values().length]; + Phaser phaser = new Phaser(numTasks * SearchPhaseName.values().length + 1); + CountDownLatch countDownLatch = new CountDownLatch(numTasks * SearchPhaseName.values().length); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + SearchPhaseContext ctx = mock(SearchPhaseContext.class); + SearchPhase mockSearchPhase = mock(SearchPhase.class); + when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase); + when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); + for (int i = 0; i < numTasks; i++) { + threads[i] = new Thread(() -> { + phaser.arriveAndAwaitAdvance(); + testRequestStats.onPhaseStart(ctx); + testRequestStats.onPhaseFailure(ctx); + countDownLatch.countDown(); + }); + threads[i].start(); + } + } + phaser.arriveAndAwaitAdvance(); + countDownLatch.await(); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + assertEquals(0, testRequestStats.getPhaseCurrent(searchPhaseName)); + } + } +} diff --git a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java index f65b0e231b1dd..c27e4bf27327a 100644 --- a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java +++ b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java @@ -32,11 +32,20 @@ package org.opensearch.index.search.stats; +import org.opensearch.action.search.SearchPhase; +import org.opensearch.action.search.SearchPhaseContext; +import org.opensearch.action.search.SearchPhaseName; +import org.opensearch.action.search.SearchRequestStats; import org.opensearch.index.search.stats.SearchStats.Stats; import org.opensearch.test.OpenSearchTestCase; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SearchStatsTests extends OpenSearchTestCase { @@ -63,6 +72,37 @@ public void testShardLevelSearchGroupStats() throws Exception { // adding again would then return wrong search stats (would return 4! instead of 3) searchStats1.add(searchStats2); assertStats(groupStats1.get("group1"), 3); + + long paramValue = randomIntBetween(2, 50); + + // Testing for request stats + SearchRequestStats testRequestStats = new SearchRequestStats(); + SearchPhaseContext ctx = mock(SearchPhaseContext.class); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + SearchPhase mockSearchPhase = mock(SearchPhase.class); + when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase); + when(mockSearchPhase.getStartTimeInNanos()).thenReturn(System.nanoTime() - TimeUnit.SECONDS.toNanos(paramValue)); + when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); + for (int iterator = 0; iterator < paramValue; iterator++) { + testRequestStats.onPhaseStart(ctx); + testRequestStats.onPhaseEnd(ctx); + } + } + searchStats1.setSearchRequestStats(testRequestStats); + for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { + assertEquals( + 0, + searchStats1.getTotal().getRequestStatsLongHolder().getRequestStatsHolder().get(searchPhaseName.getName()).current + ); + assertEquals( + paramValue, + searchStats1.getTotal().getRequestStatsLongHolder().getRequestStatsHolder().get(searchPhaseName.getName()).total + ); + assertThat( + searchStats1.getTotal().getRequestStatsLongHolder().getRequestStatsHolder().get(searchPhaseName.getName()).timeInMillis, + greaterThanOrEqualTo(paramValue) + ); + } } private static void assertStats(Stats stats, long equalTo) { @@ -87,5 +127,4 @@ private static void assertStats(Stats stats, long equalTo) { // avg_concurrency is not summed up across stats assertEquals(1, stats.getConcurrentAvgSliceCount(), 0); } - } diff --git a/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java b/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java index 9be45d4e77940..6f36d22b7e17b 100644 --- a/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java +++ b/server/src/test/java/org/opensearch/indices/NodeIndicesStatsTests.java @@ -32,6 +32,8 @@ package org.opensearch.indices; +import org.opensearch.action.admin.indices.stats.CommonStats; +import org.opensearch.action.search.SearchRequestStats; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.test.OpenSearchTestCase; @@ -43,7 +45,9 @@ public class NodeIndicesStatsTests extends OpenSearchTestCase { public void testInvalidLevel() { - final NodeIndicesStats stats = new NodeIndicesStats(null, Collections.emptyMap()); + CommonStats oldStats = new CommonStats(); + SearchRequestStats requestStats = new SearchRequestStats(); + final NodeIndicesStats stats = new NodeIndicesStats(oldStats, Collections.emptyMap(), requestStats); final String level = randomAlphaOfLength(16); final ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("level", level)); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> stats.toXContent(null, params)); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 3c31c979ce856..2b432906ee128 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2069,6 +2069,7 @@ public void onFailure(final Exception e) { new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), repositoriesServiceReference::get, fileCacheCleaner, + null, new RemoteStoreStatsTrackerFactory(clusterService, settings) ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); @@ -2297,7 +2298,8 @@ public void onFailure(final Exception e) { namedWriteableRegistry, List.of(), client - ) + ), + null ) ); actions.put( From 9f0e01743de1bf23264945d54d0d5ab20593020b Mon Sep 17 00:00:00 2001 From: Sorabh Date: Wed, 20 Sep 2023 20:32:00 -0700 Subject: [PATCH 050/388] Fix NPE in ConcurrentQueryProfile while computing the breakdown map for slices (#10111) * Fix NPE in ConcurrentQueryProfile while computing the breakdown map for slices. There can be cases where one or more slice may not have timing related information for its leaves in contexts map. During creation of slice and query level breakdown map it needs to handle such cases by using default values correctly. Also updating the min/max/avg sliceNodeTime to not include time to create weight and wait times by slice threads. It will reflect the min/max/avg execution time of each slice whereas totalNodeTime will reflect the total query time. Signed-off-by: Sorabh Hamirwasia * Address review comments Signed-off-by: Sorabh Hamirwasia --------- Signed-off-by: Sorabh Hamirwasia --- .../ConcurrentQueryProfileBreakdown.java | 84 +++++++++++--- .../ConcurrentQueryProfileBreakdownTests.java | 103 ++++++++++++++---- 2 files changed, 147 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java index f37d53737d52b..e567fdd2d436c 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; +import org.opensearch.OpenSearchException; import org.opensearch.search.profile.AbstractProfileBreakdown; import org.opensearch.search.profile.ContextualProfileBreakdown; @@ -73,14 +74,14 @@ public Map toBreakdownMap() { // If there are no leaf contexts, then return the default concurrent query level breakdown, which will include the // create_weight time/count queryNodeTime = createWeightTime; - maxSliceNodeTime = queryNodeTime; - minSliceNodeTime = queryNodeTime; - avgSliceNodeTime = queryNodeTime; + maxSliceNodeTime = 0L; + minSliceNodeTime = 0L; + avgSliceNodeTime = 0L; return buildDefaultQueryBreakdownMap(createWeightTime); } // first create the slice level breakdowns - final Map> sliceLevelBreakdowns = buildSliceLevelBreakdown(createWeightStartTime); + final Map> sliceLevelBreakdowns = buildSliceLevelBreakdown(); return buildQueryBreakdownMap(sliceLevelBreakdowns, createWeightTime, createWeightStartTime); } @@ -124,18 +125,19 @@ private Map buildDefaultQueryBreakdownMap(long createWeightTime) { /** * Computes the slice level breakdownMap. It uses sliceCollectorsToLeaves to figure out all the leaves or segments part of a slice. * Then use the breakdown timing stats for each of these leaves to calculate the breakdown stats at slice level. - * @param createWeightStartTime start time when createWeight is called + * * @return map of collector (or slice) to breakdown map */ - Map> buildSliceLevelBreakdown(long createWeightStartTime) { + Map> buildSliceLevelBreakdown() { final Map> sliceLevelBreakdowns = new HashMap<>(); - long totalSliceNodeTime = 0; + long totalSliceNodeTime = 0L; for (Map.Entry> slice : sliceCollectorsToLeaves.entrySet()) { final Collector sliceCollector = slice.getKey(); // initialize each slice level breakdown final Map currentSliceBreakdown = sliceLevelBreakdowns.computeIfAbsent(sliceCollector, k -> new HashMap<>()); // max slice end time across all timing types long sliceMaxEndTime = Long.MIN_VALUE; + long sliceMinStartTime = Long.MAX_VALUE; for (QueryTimingType timingType : QueryTimingType.values()) { if (timingType.equals(QueryTimingType.CREATE_WEIGHT)) { // do nothing for create weight as that is query level time and not slice level @@ -155,6 +157,12 @@ Map> buildSliceLevelBreakdown(long createWeightStar // leaf, but the leaf level breakdown will not be created in the contexts map. // This is because before updating the contexts map, the query hits earlyTerminationException. // To handle such case, we will ignore the leaf that is not present. + // + // Other than early termination, it can also happen in other cases. For example: there is a must boolean query + // with 2 boolean clauses. While creating scorer for first clause if no docs are found for the field in a leaf + // context then it will return null scorer. Then for 2nd clause weight as well no scorer will be created for this + // leaf context (as it is a must query). Due to this it will end up missing the leaf context in the contexts map + // for second clause weight. continue; } final Map currentSliceLeafBreakdownMap = contexts.get(sliceLeaf).toBreakdownMap(); @@ -182,15 +190,36 @@ Map> buildSliceLevelBreakdown(long createWeightStar ); } // compute sliceMaxEndTime as max of sliceEndTime across all timing types - sliceMaxEndTime = Math.max(sliceMaxEndTime, currentSliceBreakdown.get(timingTypeSliceEndTimeKey)); + sliceMaxEndTime = Math.max(sliceMaxEndTime, currentSliceBreakdown.getOrDefault(timingTypeSliceEndTimeKey, Long.MIN_VALUE)); + sliceMinStartTime = Math.min( + sliceMinStartTime, + currentSliceBreakdown.getOrDefault(timingTypeSliceStartTimeKey, Long.MAX_VALUE) + ); // compute total time for each timing type at slice level using sliceEndTime and sliceStartTime currentSliceBreakdown.put( timingType.toString(), - currentSliceBreakdown.get(timingTypeSliceEndTimeKey) - currentSliceBreakdown.get(timingTypeSliceStartTimeKey) + currentSliceBreakdown.getOrDefault(timingTypeSliceEndTimeKey, 0L) - currentSliceBreakdown.getOrDefault( + timingTypeSliceStartTimeKey, + 0L + ) ); } - // currentSliceNodeTime includes the create weight time as well which will be same for all the slices - long currentSliceNodeTime = sliceMaxEndTime - createWeightStartTime; + // currentSliceNodeTime does not include the create weight time, as that is computed in non-concurrent part + long currentSliceNodeTime; + if (sliceMinStartTime == Long.MAX_VALUE && sliceMaxEndTime == Long.MIN_VALUE) { + currentSliceNodeTime = 0L; + } else if (sliceMinStartTime == Long.MAX_VALUE || sliceMaxEndTime == Long.MIN_VALUE) { + throw new OpenSearchException( + "Unexpected value of sliceMinStartTime [" + + sliceMinStartTime + + "] or sliceMaxEndTime [" + + sliceMaxEndTime + + "] while computing the slice level timing profile breakdowns" + ); + } else { + currentSliceNodeTime = sliceMaxEndTime - sliceMinStartTime; + } + // compute max/min slice times maxSliceNodeTime = Math.max(maxSliceNodeTime, currentSliceNodeTime); minSliceNodeTime = Math.min(minSliceNodeTime, currentSliceNodeTime); @@ -245,8 +274,8 @@ public Map buildQueryBreakdownMap( // for all other timing types, we will compute min/max/avg/total across slices for (Map.Entry> sliceBreakdown : sliceLevelBreakdowns.entrySet()) { - Long sliceBreakdownTypeTime = sliceBreakdown.getValue().get(timingTypeKey); - Long sliceBreakdownTypeCount = sliceBreakdown.getValue().get(timingTypeCountKey); + long sliceBreakdownTypeTime = sliceBreakdown.getValue().getOrDefault(timingTypeKey, 0L); + long sliceBreakdownTypeCount = sliceBreakdown.getValue().getOrDefault(timingTypeCountKey, 0L); // compute max/min/avg TimingType time across slices queryBreakdownMap.compute( maxBreakdownTypeTime, @@ -276,17 +305,38 @@ public Map buildQueryBreakdownMap( ); // query start/end time for a TimingType is min/max of start/end time across slices for that TimingType - queryTimingTypeEndTime = Math.max(queryTimingTypeEndTime, sliceBreakdown.getValue().get(sliceEndTimeForTimingType)); - queryTimingTypeStartTime = Math.min(queryTimingTypeStartTime, sliceBreakdown.getValue().get(sliceStartTimeForTimingType)); + queryTimingTypeEndTime = Math.max( + queryTimingTypeEndTime, + sliceBreakdown.getValue().getOrDefault(sliceEndTimeForTimingType, Long.MIN_VALUE) + ); + queryTimingTypeStartTime = Math.min( + queryTimingTypeStartTime, + sliceBreakdown.getValue().getOrDefault(sliceStartTimeForTimingType, Long.MAX_VALUE) + ); queryTimingTypeCount += sliceBreakdownTypeCount; } + + if (queryTimingTypeStartTime == Long.MAX_VALUE || queryTimingTypeEndTime == Long.MIN_VALUE) { + throw new OpenSearchException( + "Unexpected timing type [" + + timingTypeKey + + "] start [" + + queryTimingTypeStartTime + + "] or end time [" + + queryTimingTypeEndTime + + "] computed across slices for profile results" + ); + } queryBreakdownMap.put(timingTypeKey, queryTimingTypeEndTime - queryTimingTypeStartTime); queryBreakdownMap.put(timingTypeCountKey, queryTimingTypeCount); - queryBreakdownMap.compute(avgBreakdownTypeTime, (key, value) -> (value == null) ? 0 : value / sliceLevelBreakdowns.size()); - queryBreakdownMap.compute(avgBreakdownTypeCount, (key, value) -> (value == null) ? 0 : value / sliceLevelBreakdowns.size()); + queryBreakdownMap.compute(avgBreakdownTypeTime, (key, value) -> (value == null) ? 0L : value / sliceLevelBreakdowns.size()); + queryBreakdownMap.compute(avgBreakdownTypeCount, (key, value) -> (value == null) ? 0L : value / sliceLevelBreakdowns.size()); // compute query end time using max of query end time across all timing types queryEndTime = Math.max(queryEndTime, queryTimingTypeEndTime); } + if (queryEndTime == Long.MIN_VALUE) { + throw new OpenSearchException("Unexpected error while computing the query end time across slices in profile result"); + } queryNodeTime = queryEndTime - createWeightStartTime; return queryBreakdownMap; } diff --git a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java index 065c90b262e11..f29ba3b0cea07 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java @@ -81,9 +81,9 @@ public void testBreakdownMapWithNoLeafContext() throws Exception { ); // verify total/min/max/avg node time is same as weight time assertEquals(createWeightTime, testQueryProfileBreakdown.toNodeTime()); - assertEquals(createWeightTime, testQueryProfileBreakdown.getMaxSliceNodeTime()); - assertEquals(createWeightTime, testQueryProfileBreakdown.getMinSliceNodeTime()); - assertEquals(createWeightTime, testQueryProfileBreakdown.getAvgSliceNodeTime()); + assertEquals(0, testQueryProfileBreakdown.getMaxSliceNodeTime()); + assertEquals(0, testQueryProfileBreakdown.getMinSliceNodeTime()); + assertEquals(0, testQueryProfileBreakdown.getAvgSliceNodeTime()); continue; } assertEquals(0, (long) queryBreakDownMap.get(timingTypeKey)); @@ -103,16 +103,15 @@ public void testBuildSliceLevelBreakdownWithSingleSlice() throws Exception { final LeafReaderContext sliceLeaf = directoryReader.leaves().get(0); final Collector sliceCollector = mock(Collector.class); final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); - final Map leafProfileBreakdownMap = getLeafBreakdownMap(createWeightEarliestStartTime + 10, 10, 1); + final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); + final Map leafProfileBreakdownMap = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); final AbstractProfileBreakdown leafProfileBreakdown = new TestQueryProfileBreakdown( QueryTimingType.class, leafProfileBreakdownMap ); testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector, sliceLeaf); testQueryProfileBreakdown.getContexts().put(sliceLeaf, leafProfileBreakdown); - final Map> sliceBreakdownMap = testQueryProfileBreakdown.buildSliceLevelBreakdown( - createWeightEarliestStartTime - ); + final Map> sliceBreakdownMap = testQueryProfileBreakdown.buildSliceLevelBreakdown(); assertFalse(sliceBreakdownMap == null || sliceBreakdownMap.isEmpty()); assertEquals(1, sliceBreakdownMap.size()); assertTrue(sliceBreakdownMap.containsKey(sliceCollector)); @@ -141,9 +140,9 @@ public void testBuildSliceLevelBreakdownWithSingleSlice() throws Exception { (long) sliceBreakdown.get(timingTypeKey + SLICE_END_TIME_SUFFIX) ); } - assertEquals(20, testQueryProfileBreakdown.getMaxSliceNodeTime()); - assertEquals(20, testQueryProfileBreakdown.getMinSliceNodeTime()); - assertEquals(20, testQueryProfileBreakdown.getAvgSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdown.getMaxSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdown.getMinSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdown.getAvgSliceNodeTime()); directoryReader.close(); directory.close(); } @@ -154,8 +153,9 @@ public void testBuildSliceLevelBreakdownWithMultipleSlices() throws Exception { final Collector sliceCollector_1 = mock(Collector.class); final Collector sliceCollector_2 = mock(Collector.class); final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); - final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEarliestStartTime + 10, 10, 1); - final Map leafProfileBreakdownMap_2 = getLeafBreakdownMap(createWeightEarliestStartTime + 40, 10, 1); + final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); + final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); + final Map leafProfileBreakdownMap_2 = getLeafBreakdownMap(createWeightEndTime + 40, 10, 1); final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown( QueryTimingType.class, leafProfileBreakdownMap_1 @@ -168,9 +168,7 @@ public void testBuildSliceLevelBreakdownWithMultipleSlices() throws Exception { testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_2, directoryReader.leaves().get(1)); testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(0), leafProfileBreakdown_1); testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(1), leafProfileBreakdown_2); - final Map> sliceBreakdownMap = testQueryProfileBreakdown.buildSliceLevelBreakdown( - createWeightEarliestStartTime - ); + final Map> sliceBreakdownMap = testQueryProfileBreakdown.buildSliceLevelBreakdown(); assertFalse(sliceBreakdownMap == null || sliceBreakdownMap.isEmpty()); assertEquals(2, sliceBreakdownMap.size()); @@ -208,9 +206,9 @@ public void testBuildSliceLevelBreakdownWithMultipleSlices() throws Exception { } } - assertEquals(50, testQueryProfileBreakdown.getMaxSliceNodeTime()); - assertEquals(20, testQueryProfileBreakdown.getMinSliceNodeTime()); - assertEquals(35, testQueryProfileBreakdown.getAvgSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdown.getMaxSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdown.getMinSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdown.getAvgSliceNodeTime()); directoryReader.close(); directory.close(); } @@ -221,8 +219,9 @@ public void testBreakDownMapWithMultipleSlices() throws Exception { final Collector sliceCollector_1 = mock(Collector.class); final Collector sliceCollector_2 = mock(Collector.class); final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); - final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEarliestStartTime + 10, 10, 1); - final Map leafProfileBreakdownMap_2 = getLeafBreakdownMap(createWeightEarliestStartTime + 40, 20, 1); + final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); + final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); + final Map leafProfileBreakdownMap_2 = getLeafBreakdownMap(createWeightEndTime + 40, 20, 1); final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown( QueryTimingType.class, leafProfileBreakdownMap_1 @@ -269,9 +268,67 @@ public void testBreakDownMapWithMultipleSlices() throws Exception { assertEquals(1, (long) queryBreakDownMap.get(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeCountKey)); } - assertEquals(60, testQueryProfileBreakdown.getMaxSliceNodeTime()); - assertEquals(20, testQueryProfileBreakdown.getMinSliceNodeTime()); - assertEquals(40, testQueryProfileBreakdown.getAvgSliceNodeTime()); + assertEquals(20, testQueryProfileBreakdown.getMaxSliceNodeTime()); + assertEquals(10, testQueryProfileBreakdown.getMinSliceNodeTime()); + assertEquals(15, testQueryProfileBreakdown.getAvgSliceNodeTime()); + directoryReader.close(); + directory.close(); + } + + public void testBreakDownMapWithMultipleSlicesAndOneSliceWithNoLeafContext() throws Exception { + final DirectoryReader directoryReader = getDirectoryReader(2); + final Directory directory = directoryReader.directory(); + final Collector sliceCollector_1 = mock(Collector.class); + final Collector sliceCollector_2 = mock(Collector.class); + final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime(); + final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming(); + final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1); + final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown( + QueryTimingType.class, + leafProfileBreakdownMap_1 + ); + testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_1, directoryReader.leaves().get(0)); + testQueryProfileBreakdown.associateCollectorToLeaves(sliceCollector_2, directoryReader.leaves().get(1)); + testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(0), leafProfileBreakdown_1); + // leaf2 profile breakdown is not present in contexts map + + Map queryBreakDownMap = testQueryProfileBreakdown.toBreakdownMap(); + assertFalse(queryBreakDownMap == null || queryBreakDownMap.isEmpty()); + assertEquals(66, queryBreakDownMap.size()); + + for (QueryTimingType queryTimingType : QueryTimingType.values()) { + String timingTypeKey = queryTimingType.toString(); + String timingTypeCountKey = queryTimingType + TIMING_TYPE_COUNT_SUFFIX; + + if (queryTimingType.equals(QueryTimingType.CREATE_WEIGHT)) { + final long createWeightTime = queryBreakDownMap.get(timingTypeKey); + assertEquals(createWeightTimer.getApproximateTiming(), createWeightTime); + assertEquals(1, (long) queryBreakDownMap.get(timingTypeCountKey)); + // verify there is no min/max/avg for weight type stats + assertFalse( + queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeKey) + || queryBreakDownMap.containsKey(MIN_PREFIX + timingTypeKey) + || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeKey) + || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeCountKey) + || queryBreakDownMap.containsKey(MIN_PREFIX + timingTypeCountKey) + || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeCountKey) + ); + continue; + } + assertEquals(10, (long) queryBreakDownMap.get(timingTypeKey)); + assertEquals(10, (long) queryBreakDownMap.get(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeKey)); + assertEquals(5, (long) queryBreakDownMap.get(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeKey)); + assertEquals(0, (long) queryBreakDownMap.get(MIN_PREFIX + timingTypeKey)); + assertEquals(1, (long) queryBreakDownMap.get(timingTypeCountKey)); + assertEquals(1, (long) queryBreakDownMap.get(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeCountKey)); + // min of 0 means one of the slice didn't worked on any leaf context + assertEquals(0, (long) queryBreakDownMap.get(MIN_PREFIX + timingTypeCountKey)); + assertEquals(0, (long) queryBreakDownMap.get(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeCountKey)); + } + + assertEquals(10, testQueryProfileBreakdown.getMaxSliceNodeTime()); + assertEquals(0, testQueryProfileBreakdown.getMinSliceNodeTime()); + assertEquals(5, testQueryProfileBreakdown.getAvgSliceNodeTime()); directoryReader.close(); directory.close(); } From ef343d769439f510aba6f20b7746bbcf5f42e377 Mon Sep 17 00:00:00 2001 From: Poojita Raj Date: Thu, 21 Sep 2023 06:20:23 -0700 Subject: [PATCH 051/388] Bump version of jgit to 6.7.0.202309050840-r to fix CVE-2023-4759 (#10147) * change dependency version of jgit Signed-off-by: Poojita Raj * add changelog Signed-off-by: Poojita Raj --------- Signed-off-by: Poojita Raj --- CHANGELOG.md | 3 ++- buildSrc/build.gradle | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10205ae20c001..18b675c264718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247)) - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963)) - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822)) +- Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) @@ -110,4 +111,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 3dc689795151b..8fbd7cf7770fb 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -133,6 +133,10 @@ dependencies { integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') { exclude module: "groovy" } + implementation('org.ajoberstar.grgit:grgit-core:5.2.0') { + exclude group: 'org.eclipse.jgit', module: 'org.eclipse.jgit' + } + implementation 'org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r' } configurations.all { From 7dca3ca82576cd1e16ba23aabf3d781aa35d210b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 09:51:05 -0400 Subject: [PATCH 052/388] Bump com.github.tomakehurst:wiremock-jre8-standalone from 2.35.0 to 3.0.1 in /buildSrc (#9752) * Bump com.github.tomakehurst:wiremock-jre8-standalone in /buildSrc Bumps [com.github.tomakehurst:wiremock-jre8-standalone](https://github.com/wiremock/wiremock) from 2.35.0 to 3.0.1. - [Release notes](https://github.com/wiremock/wiremock/releases) - [Commits](https://github.com/wiremock/wiremock/compare/2.35.0...3.0.1) --- updated-dependencies: - dependency-name: com.github.tomakehurst:wiremock-jre8-standalone dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] * Bump org.wiremock:wiremock-standalone in /buildSrc Signed-off-by: Andriy Redko --------- Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18b675c264718..8bdc8b665217d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `bytebuddy` from 1.14.3 to 1.14.7 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022)) - Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098)) - Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125)) +- Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 8fbd7cf7770fb..d7bdd09ea882e 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -128,7 +128,7 @@ dependencies { testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" testFixturesApi gradleApi() testFixturesApi gradleTestKit() - testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.35.0' + testImplementation 'org.wiremock:wiremock-standalone:3.1.0' testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}" integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') { exclude module: "groovy" From a63f8def960164117a57feb79911df4629415875 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 21 Sep 2023 11:09:11 -0700 Subject: [PATCH 053/388] [Feature] Visitor design pattern in QueryBuilder (#10110) * Implementation of Visitor Design Pattern in QueryBuilder Signed-off-by: Varun Jain * Adding Changelog Signed-off-by: Varun Jain * Adding Changelog Signed-off-by: Varun Jain * Comments addressed of michael Signed-off-by: Varun Jain * Adding test case and some minor fixes in DisMaxQueryBuilder Signed-off-by: Varun Jain * Modifying test cases and remove exeception Signed-off-by: Varun Jain * Adding bool query builder test cases at clause level Signed-off-by: Varun Jain * Fixing test case Signed-off-by: Varun Jain * Additional test cases Signed-off-by: Varun Jain --------- Signed-off-by: Varun Jain --- CHANGELOG.md | 1 + .../index/query/BoolQueryBuilder.java | 31 ++++++++++++ .../index/query/BoostingQueryBuilder.java | 12 +++++ .../query/ConstantScoreQueryBuilder.java | 8 ++++ .../index/query/DisMaxQueryBuilder.java | 12 +++++ .../query/FieldMaskingSpanQueryBuilder.java | 7 +++ .../opensearch/index/query/QueryBuilder.java | 9 ++++ .../index/query/QueryBuilderVisitor.java | 48 +++++++++++++++++++ .../query/SpanContainingQueryBuilder.java | 8 ++++ .../index/query/SpanFirstQueryBuilder.java | 7 +++ .../query/SpanMultiTermQueryBuilder.java | 9 ++++ .../index/query/SpanNearQueryBuilder.java | 12 +++++ .../index/query/SpanNotQueryBuilder.java | 13 +++++ .../index/query/SpanOrQueryBuilder.java | 12 +++++ .../index/query/SpanWithinQueryBuilder.java | 8 ++++ .../index/query/BoolQueryBuilderTests.java | 25 ++++++++++ .../query/BoostingQueryBuilderTests.java | 14 ++++++ .../query/ConstantScoreQueryBuilderTests.java | 10 ++++ .../index/query/DisMaxQueryBuilderTests.java | 11 +++++ .../FieldMaskingSpanQueryBuilderTests.java | 8 ++++ .../index/query/QueryBuilderVisitorTests.java | 33 +++++++++++++ .../SpanContainingQueryBuilderTests.java | 8 ++++ .../query/SpanFirstQueryBuilderTests.java | 8 ++++ .../query/SpanMultiTermQueryBuilderTests.java | 10 ++++ .../query/SpanNearQueryBuilderTests.java | 10 ++++ .../index/query/SpanNotQueryBuilderTests.java | 8 ++++ .../index/query/SpanOrQueryBuilderTests.java | 10 ++++ .../query/SpanWithinQueryBuilderTests.java | 8 ++++ .../test/AbstractBuilderTestCase.java | 17 +++++++ 29 files changed, 377 insertions(+) create mode 100644 server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java create mode 100644 server/src/test/java/org/opensearch/index/query/QueryBuilderVisitorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bdc8b665217d..a90a07bf695fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107)) - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679)) +- Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 diff --git a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java index 65c2dfa9c5a8b..c44a7ef6a397c 100644 --- a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java @@ -427,4 +427,35 @@ private static boolean rewriteClauses( } return changed; } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (mustClauses.isEmpty() == false) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(Occur.MUST); + for (QueryBuilder mustClause : mustClauses) { + mustClause.visit(subVisitor); + } + } + if (shouldClauses.isEmpty() == false) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(Occur.SHOULD); + for (QueryBuilder shouldClause : shouldClauses) { + shouldClause.visit(subVisitor); + } + } + if (mustNotClauses.isEmpty() == false) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(Occur.MUST_NOT); + for (QueryBuilder mustNotClause : mustNotClauses) { + mustNotClause.visit(subVisitor); + } + } + if (filterClauses.isEmpty() == false) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(Occur.FILTER); + for (QueryBuilder filterClause : filterClauses) { + filterClause.visit(subVisitor); + } + } + + } + } diff --git a/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java index 26124b422f26f..1b52ae2f03605 100644 --- a/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java @@ -33,6 +33,7 @@ package org.opensearch.index.query; import org.apache.lucene.queries.function.FunctionScoreQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -252,4 +253,15 @@ protected void extractInnerHitBuilders(Map inner InnerHitContextBuilder.extractInnerHits(positiveQuery, innerHits); InnerHitContextBuilder.extractInnerHits(negativeQuery, innerHits); } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (positiveQuery != null) { + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(positiveQuery); + } + if (negativeQuery != null) { + visitor.getChildVisitor(BooleanClause.Occur.SHOULD).accept(negativeQuery); + } + } } diff --git a/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java index 6a29ad8a0a401..b2764d29da80a 100644 --- a/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java @@ -32,6 +32,7 @@ package org.opensearch.index.query; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; @@ -183,4 +184,11 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws protected void extractInnerHitBuilders(Map innerHits) { InnerHitContextBuilder.extractInnerHits(filterBuilder, innerHits); } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + visitor.getChildVisitor(BooleanClause.Occur.FILTER).accept(filterBuilder); + } + } diff --git a/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java index 91f4a02fac6c0..bd8ec62f6c43e 100644 --- a/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java @@ -32,6 +32,7 @@ package org.opensearch.index.query; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.Query; import org.opensearch.common.lucene.search.Queries; @@ -246,4 +247,15 @@ protected void extractInnerHitBuilders(Map inner InnerHitContextBuilder.extractInnerHits(query, innerHits); } } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (queries.isEmpty() == false) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(BooleanClause.Occur.SHOULD); + for (QueryBuilder subQb : queries) { + subVisitor.accept(subQb); + } + } + } } diff --git a/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java index 09a71795a3f27..1162689a54689 100644 --- a/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java @@ -34,6 +34,7 @@ import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -207,4 +208,10 @@ protected boolean doEquals(FieldMaskingSpanQueryBuilder other) { public String getWriteableName() { return SPAN_FIELD_MASKING_FIELD.getPreferredName(); } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(queryBuilder); + } } diff --git a/server/src/main/java/org/opensearch/index/query/QueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryBuilder.java index a40ccf427794a..090f74c5be7fe 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/QueryBuilder.java @@ -95,4 +95,13 @@ public interface QueryBuilder extends NamedWriteable, ToXContentObject, Rewritea default QueryBuilder rewrite(QueryRewriteContext queryShardContext) throws IOException { return this; } + + /** + * Recurse through the QueryBuilder tree, visiting any child QueryBuilder. + * @param visitor a query builder visitor to be called by each query builder in the tree. + */ + default void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + }; + } diff --git a/server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java b/server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java new file mode 100644 index 0000000000000..af5a125f9dd95 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.query; + +import org.apache.lucene.search.BooleanClause; + +/** + * QueryBuilderVisitor is an interface to define Visitor Object to be traversed in QueryBuilder tree. + */ +public interface QueryBuilderVisitor { + + /** + * Accept method is called when the visitor accepts the queryBuilder object to be traversed in the query tree. + * @param qb is a queryBuilder object which is accepeted by the visitor. + */ + void accept(QueryBuilder qb); + + /** + * Fetches the child sub visitor from the main QueryBuilderVisitor Object. + * @param occur defines the occurrence of the result fetched from the search query in the final search result. + * @return a child queryBuilder Visitor Object. + */ + QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur); + + /** + * NoopQueryVisitor is a default implementation of QueryBuilderVisitor. + * When a user does not want to implement QueryBuilderVisitor and have to just pass an empty object then this class will be used. + * + */ + QueryBuilderVisitor NO_OP_VISITOR = new QueryBuilderVisitor() { + @Override + public void accept(QueryBuilder qb) { + // Do nothing + } + + @Override + public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) { + return this; + } + }; + +} diff --git a/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java index ed4f5c6848b06..32a19ea3e9b50 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java @@ -34,6 +34,7 @@ import org.apache.lucene.queries.spans.SpanContainingQuery; import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -188,4 +189,11 @@ protected boolean doEquals(SpanContainingQueryBuilder other) { public String getWriteableName() { return NAME; } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(big); + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(little); + } } diff --git a/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java index 7427b13463284..bcbc64ddf386d 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java @@ -34,6 +34,7 @@ import org.apache.lucene.queries.spans.SpanFirstQuery; import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -186,4 +187,10 @@ protected boolean doEquals(SpanFirstQueryBuilder other) { public String getWriteableName() { return NAME; } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(matchBuilder); + } } diff --git a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java index ce28391ca478b..96d03c91964e3 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java @@ -33,6 +33,7 @@ import org.apache.lucene.queries.SpanMatchNoDocsQuery; import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -213,4 +214,12 @@ protected boolean doEquals(SpanMultiTermQueryBuilder other) { public String getWriteableName() { return NAME; } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (multiTermQueryBuilder != null) { + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(multiTermQueryBuilder); + } + } } diff --git a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java index ba7625d94a5a6..30a1c29c29126 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java @@ -34,6 +34,7 @@ import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -299,6 +300,17 @@ public String getWriteableName() { return NAME; } + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (this.clauses.isEmpty() == false) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(BooleanClause.Occur.MUST); + for (QueryBuilder subQb : this.clauses) { + subVisitor.accept(subQb); + } + } + } + /** * SpanGapQueryBuilder enables gaps in a SpanNearQuery. * Since, SpanGapQuery is private to SpanNearQuery, SpanGapQueryBuilder cannot diff --git a/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java index 98e7f287749f5..59ec5b9d77fc8 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java @@ -34,6 +34,7 @@ import org.apache.lucene.queries.spans.SpanNotQuery; import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -284,4 +285,16 @@ protected boolean doEquals(SpanNotQueryBuilder other) { public String getWriteableName() { return NAME; } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (include != null) { + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(include); + } + + if (exclude != null) { + visitor.getChildVisitor(BooleanClause.Occur.MUST_NOT).accept(exclude); + } + } } diff --git a/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java index 2f63e6d7403f7..fae1e318c66bd 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java @@ -34,6 +34,7 @@ import org.apache.lucene.queries.spans.SpanOrQuery; import org.apache.lucene.queries.spans.SpanQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -188,4 +189,15 @@ protected boolean doEquals(SpanOrQueryBuilder other) { public String getWriteableName() { return NAME; } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (clauses.isEmpty() == false) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(BooleanClause.Occur.SHOULD); + for (QueryBuilder subQb : this.clauses) { + subVisitor.accept(subQb); + } + } + } } diff --git a/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java index 5d02cc0026dfd..4d5a6dde61a70 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java @@ -34,6 +34,7 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanWithinQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -197,4 +198,11 @@ protected boolean doEquals(SpanWithinQueryBuilder other) { public String getWriteableName() { return NAME; } + + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(big); + visitor.getChildVisitor(BooleanClause.Occur.MUST).accept(little); + } } diff --git a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java index 2160ab6220866..d0f26f3026789 100644 --- a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java @@ -47,10 +47,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; @@ -456,4 +459,26 @@ public void testMustRewrite() throws IOException { IllegalStateException e = expectThrows(IllegalStateException.class, () -> boolQuery.toQuery(context)); assertEquals("Rewrite first", e.getMessage()); } + + public void testVisit() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new TermQueryBuilder(TEXT_FIELD_NAME, "should")); + boolQueryBuilder.must(new TermQueryBuilder(TEXT_FIELD_NAME, "must1")); + boolQueryBuilder.must(new TermQueryBuilder(TEXT_FIELD_NAME, "must2")); // Add a second one to confirm that they both get visited + boolQueryBuilder.mustNot(new TermQueryBuilder(TEXT_FIELD_NAME, "mustNot")); + boolQueryBuilder.filter(new TermQueryBuilder(TEXT_FIELD_NAME, "filter")); + List visitedQueries = new ArrayList<>(); + boolQueryBuilder.visit(createTestVisitor(visitedQueries)); + assertEquals(6, visitedQueries.size()); + Set set = new HashSet<>(Arrays.asList("should", "must1", "must2", "mustNot", "filter")); + + for (QueryBuilder qb : visitedQueries) { + if (qb instanceof TermQueryBuilder) { + set.remove(((TermQueryBuilder) qb).value()); + } + } + + assertEquals(0, set.size()); + + } } diff --git a/server/src/test/java/org/opensearch/index/query/BoostingQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/BoostingQueryBuilderTests.java index 94ded24975be4..66a02a02d4e5b 100644 --- a/server/src/test/java/org/opensearch/index/query/BoostingQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/BoostingQueryBuilderTests.java @@ -38,6 +38,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.nullValue; @@ -153,4 +155,16 @@ public void testMustRewrite() throws IOException { e = expectThrows(IllegalStateException.class, () -> queryBuilder2.toQuery(context)); assertEquals("Rewrite first", e.getMessage()); } + + public void testVisit() { + BoostingQueryBuilder builder = new BoostingQueryBuilder( + new TermQueryBuilder("unmapped_field", "value"), + new TermQueryBuilder(KEYWORD_FIELD_NAME, "other_value") + ); + + List visitedQueries = new ArrayList<>(); + builder.visit(createTestVisitor(visitedQueries)); + + assertEquals(3, visitedQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java index 2bfe964ce7259..527413d2513d0 100644 --- a/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/ConstantScoreQueryBuilderTests.java @@ -39,6 +39,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.nullValue; @@ -133,4 +135,12 @@ public void testMustRewrite() throws IOException { IllegalStateException e = expectThrows(IllegalStateException.class, () -> queryBuilder.toQuery(context)); assertEquals("Rewrite first", e.getMessage()); } + + public void testVisit() { + ConstantScoreQueryBuilder queryBuilder = new ConstantScoreQueryBuilder(new TermQueryBuilder("unmapped_field", "foo")); + List visitorQueries = new ArrayList<>(); + queryBuilder.visit(createTestVisitor(visitorQueries)); + + assertEquals(2, visitorQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java index 8d589bd76f2bb..cb0df38de5c02 100644 --- a/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java @@ -41,6 +41,7 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -156,4 +157,14 @@ public void testRewriteMultipleTimes() throws IOException { assertEquals(rewrittenAgain, expected); assertEquals(Rewriteable.rewrite(dismax, createShardContext()), expected); } + + public void testVisit() { + DisMaxQueryBuilder dismax = new DisMaxQueryBuilder(); + dismax.add(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()).toString())); + + List visitedQueries = new ArrayList<>(); + dismax.visit(createTestVisitor(visitedQueries)); + + assertEquals(2, visitedQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java index 7e893a0947531..db0a7bf1795ff 100644 --- a/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -41,6 +41,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.opensearch.index.query.FieldMaskingSpanQueryBuilder.SPAN_FIELD_MASKING_FIELD; import static org.hamcrest.CoreMatchers.equalTo; @@ -147,4 +149,10 @@ public void testDeprecatedName() throws IOException { "Deprecated field [field_masking_span] used, expected [" + SPAN_FIELD_MASKING_FIELD.getPreferredName() + "] instead" ); } + + public void testVisit() { + List visitedQueries = new ArrayList<>(); + doCreateTestQueryBuilder().visit(createTestVisitor(visitedQueries)); + assertEquals(2, visitedQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/QueryBuilderVisitorTests.java b/server/src/test/java/org/opensearch/index/query/QueryBuilderVisitorTests.java new file mode 100644 index 0000000000000..7849d3985ca59 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/query/QueryBuilderVisitorTests.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.query; + +import org.apache.lucene.search.BooleanClause; +import org.opensearch.test.AbstractBuilderTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class QueryBuilderVisitorTests extends AbstractBuilderTestCase { + + public void testNoOpsVisitor() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + + List visitedQueries = new ArrayList<>(); + QueryBuilderVisitor qbv = createTestVisitor(visitedQueries); + boolQueryBuilder.visit(qbv); + QueryBuilderVisitor subQbv = qbv.getChildVisitor(BooleanClause.Occur.MUST_NOT); + assertEquals(0, visitedQueries.size()); + assertEquals(qbv, subQbv); + } + + protected static QueryBuilderVisitor createTestVisitor(List visitedQueries) { + return QueryBuilderVisitor.NO_OP_VISITOR; + } +} diff --git a/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java index fff4369e155b3..2a5441b0ea932 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanContainingQueryBuilderTests.java @@ -38,6 +38,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -192,4 +194,10 @@ public void testFromJsonWithNonDefaultBoostInLittleQuery() { equalTo("span_containing [little] as a nested span clause can't have non-default boost value [2.0]") ); } + + public void testVisit() { + List visitorQueries = new ArrayList<>(); + doCreateTestQueryBuilder().visit(createTestVisitor(visitorQueries)); + assertEquals(3, visitorQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java index 781d2defcfd37..82bb77726b037 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanFirstQueryBuilderTests.java @@ -40,6 +40,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.opensearch.index.query.QueryBuilders.spanTermQuery; import static org.hamcrest.CoreMatchers.equalTo; @@ -128,4 +130,10 @@ public void testFromJsonWithNonDefaultBoostInMatchQuery() { Exception exception = expectThrows(ParsingException.class, () -> parseQuery(json)); assertThat(exception.getMessage(), equalTo("span_first [match] as a nested span clause can't have non-default boost value [2.0]")); } + + public void testVisit() { + List visitorQueries = new ArrayList<>(); + doCreateTestQueryBuilder().visit(createTestVisitor(visitorQueries)); + assertEquals(2, visitorQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java index e20d56c6aff2a..b4abff118802e 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java @@ -60,6 +60,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static java.util.Collections.singleton; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; @@ -306,4 +308,12 @@ public void testTopNMultiTermsRewriteInsideSpan() throws Exception { } } + + public void testVisit() { + MultiTermQueryBuilder multiTermQueryBuilder = new PrefixQueryBuilderTests().createTestQueryBuilder(); + SpanMultiTermQueryBuilder spanMultiTermQueryBuilder = new SpanMultiTermQueryBuilder(multiTermQueryBuilder); + List visitorQueries = new ArrayList<>(); + spanMultiTermQueryBuilder.visit(createTestVisitor(visitorQueries)); + assertEquals(2, visitorQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java index 8c9130d4b7bbd..c97b541da2199 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanNearQueryBuilderTests.java @@ -41,7 +41,9 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.either; @@ -231,4 +233,12 @@ public void testFromJsonWithNonDefaultBoostInInnerQuery() { Exception exception = expectThrows(ParsingException.class, () -> parseQuery(json)); assertThat(exception.getMessage(), equalTo("span_near [clauses] as a nested span clause can't have non-default boost value [2.0]")); } + + public void testVisit() { + SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(1); + SpanNearQueryBuilder spanNearQueryBuilder = new SpanNearQueryBuilder(spanTermQueries[0], 1); + List visitorQueries = new ArrayList<>(); + spanNearQueryBuilder.visit(createTestVisitor(visitorQueries)); + assertEquals(2, visitorQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java index b85c7cf81b0e1..ff42b271ffca5 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanNotQueryBuilderTests.java @@ -40,6 +40,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.opensearch.index.query.QueryBuilders.spanNearQuery; import static org.opensearch.index.query.QueryBuilders.spanTermQuery; @@ -316,4 +318,10 @@ public void testFromJsonWithNonDefaultBoostInExcludeQuery() { Exception exception = expectThrows(ParsingException.class, () -> parseQuery(json)); assertThat(exception.getMessage(), equalTo("span_not [exclude] as a nested span clause can't have non-default boost value [2.0]")); } + + public void testVisit() { + List visitedQueries = new ArrayList<>(); + doCreateTestQueryBuilder().visit(createTestVisitor(visitedQueries)); + assertEquals(3, visitedQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java index 45323b5df74df..eb4b8fd486cb0 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanOrQueryBuilderTests.java @@ -39,7 +39,9 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -137,4 +139,12 @@ public void testFromJsonWithNonDefaultBoostInInnerQuery() { Exception exception = expectThrows(ParsingException.class, () -> parseQuery(json)); assertThat(exception.getMessage(), equalTo("span_or [clauses] as a nested span clause can't have non-default boost value [2.0]")); } + + public void testVisit() { + SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder("demo", "demo"); + SpanOrQueryBuilder spanOrQueryBuilder = new SpanOrQueryBuilder(spanTermQueryBuilder); + List visitedQueries = new ArrayList<>(); + spanOrQueryBuilder.visit(createTestVisitor(visitedQueries)); + assertEquals(2, visitedQueries.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java index e8b6a21254ff8..74b955e872d51 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanWithinQueryBuilderTests.java @@ -38,6 +38,8 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -189,4 +191,10 @@ public void testFromJsonWithNonDefaultBoostInLittleQuery() { equalTo("span_within [little] as a nested span clause can't have non-default boost value [2.0]") ); } + + public void testVisit() { + List visitedQueries = new ArrayList<>(); + doCreateTestQueryBuilder().visit(createTestVisitor(visitedQueries)); + assertEquals(3, visitedQueries.size()); + } } diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java index 0cfa64df55659..7a8cf4963c4f1 100644 --- a/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/AbstractBuilderTestCase.java @@ -35,6 +35,7 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.SeedUtils; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.Accountable; import org.opensearch.Version; @@ -69,6 +70,8 @@ import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilderVisitor; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.IndicesModule; @@ -316,6 +319,20 @@ protected static QueryShardContext createShardContext() { return createShardContext(null); } + protected static QueryBuilderVisitor createTestVisitor(List visitedQueries) { + return new QueryBuilderVisitor() { + @Override + public void accept(QueryBuilder qb) { + visitedQueries.add(qb); + } + + @Override + public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) { + return this; + } + }; + } + private static class ClientInvocationHandler implements InvocationHandler { AbstractBuilderTestCase delegate; From 3b9c0050aff4777529e25157ed490197bdc6217b Mon Sep 17 00:00:00 2001 From: shephali mittal Date: Fri, 22 Sep 2023 01:01:05 +0530 Subject: [PATCH 054/388] [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter (#9666) * [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter Signed-off-by: Shephali Mittal * Made endpoint settings configurable in Exporter Factory Signed-off-by: Shephali Mittal * Added comments Signed-off-by: Shephali Mittal * precommit fix Signed-off-by: Shephali Mittal * OTelTelemetryPluginTests test fix Signed-off-by: Shephali Mittal * Added test for OtlpGrpcSpanExporterProvider Signed-off-by: Shephali Mittal * renamed var createMethodContainsSettingsParam Signed-off-by: Shephali Mittal * modified endpoint comment Signed-off-by: Shephali Mittal * Remove configurable settings and provided support for getDefault() Signed-off-by: Shephali Mittal * Updated dependencies Signed-off-by: Shephali Mittal * Added opentelemetry-exporter-sender-okhttp dependency Signed-off-by: Shephali Mittal * Made opentelemetry-exporter-sender-okhttp dependency runtimeonly Signed-off-by: Shephali Mittal * Added license files for opentelemetry-exporter-sender-okhttp Signed-off-by: Shephali Mittal * Updated okhttp dependencies to runtime only Signed-off-by: Shephali Mittal --------- Signed-off-by: Shephali Mittal Signed-off-by: shephali mittal Co-authored-by: Shephali Mittal --- CHANGELOG.md | 1 + plugins/telemetry-otel/build.gradle | 36 +++- .../licenses/kotlin-stdlib-1.7.10.jar.sha1 | 1 + .../licenses/kotlin-stdlib-LICENSE.txt | 202 +++++++++++++++++ .../licenses/kotlin-stdlib-NOTICE.txt | 0 .../licenses/okhttp-4.11.0.jar.sha1 | 1 + .../licenses/okhttp-LICENSE.txt | 202 +++++++++++++++++ .../telemetry-otel/licenses/okhttp-NOTICE.txt | 0 .../licenses/okio-jvm-3.5.0.jar.sha1 | 1 + .../licenses/okio-jvm-LICENSE.txt | 202 +++++++++++++++++ .../licenses/okio-jvm-NOTICE.txt | 0 ...ntelemetry-exporter-common-1.30.1.jar.sha1 | 1 + .../opentelemetry-exporter-common-LICENSE.txt | 202 +++++++++++++++++ .../opentelemetry-exporter-common-NOTICE.txt | 0 ...pentelemetry-exporter-otlp-1.30.1.jar.sha1 | 1 + .../opentelemetry-exporter-otlp-LICENSE.txt | 202 +++++++++++++++++ .../opentelemetry-exporter-otlp-NOTICE.txt | 0 ...metry-exporter-otlp-common-1.30.1.jar.sha1 | 1 + ...telemetry-exporter-otlp-common-LICENSE.txt | 202 +++++++++++++++++ ...ntelemetry-exporter-otlp-common-NOTICE.txt | 0 ...try-exporter-sender-okhttp-1.30.1.jar.sha1 | 1 + ...lemetry-exporter-sender-okhttp-LICENSE.txt | 203 ++++++++++++++++++ ...elemetry-exporter-sender-okhttp-NOTICE.txt | 0 .../tracing/OTelResourceProvider.java | 14 +- .../exporter/OTelSpanExporterFactory.java | 11 +- .../plugin-metadata/plugin-security.policy | 2 + .../DummySpanExporterWithGetDefault.java | 37 ++++ .../OTelSpanExporterFactoryTests.java | 11 + 28 files changed, 1527 insertions(+), 7 deletions(-) create mode 100644 plugins/telemetry-otel/licenses/kotlin-stdlib-1.7.10.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/kotlin-stdlib-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/kotlin-stdlib-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/okhttp-4.11.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/okhttp-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/okhttp-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/okio-jvm-3.5.0.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/okio-jvm-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/okio-jvm-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-NOTICE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-NOTICE.txt create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporterWithGetDefault.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a90a07bf695fe..6c1108cdf3ee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add coordinator level stats for search latency ([#8386](https://github.com/opensearch-project/OpenSearch/issues/8386)) - Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681)) - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694)) +- [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/plugins/telemetry-otel/build.gradle b/plugins/telemetry-otel/build.gradle index 511bae590bf13..45c9f522c09d8 100644 --- a/plugins/telemetry-otel/build.gradle +++ b/plugins/telemetry-otel/build.gradle @@ -30,6 +30,13 @@ dependencies { api "io.opentelemetry:opentelemetry-exporter-logging:${versions.opentelemetry}" api "io.opentelemetry:opentelemetry-semconv:${versions.opentelemetry}-alpha" api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-exporter-otlp:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-exporter-common:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-exporter-otlp-common:${versions.opentelemetry}" + runtimeOnly "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" + runtimeOnly "com.squareup.okhttp3:okhttp:4.11.0" + runtimeOnly "com.squareup.okio:okio-jvm:3.5.0" + runtimeOnly "io.opentelemetry:opentelemetry-exporter-sender-okhttp:${versions.opentelemetry}" testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${versions.opentelemetry}" } @@ -44,6 +51,32 @@ thirdPartyAudit { ) ignoreMissingClasses( + 'android.net.http.X509TrustManagerExtensions', + 'android.net.ssl.SSLSockets', + 'android.os.Build$VERSION', + 'android.security.NetworkSecurityPolicy', + 'android.util.Log', + 'com.google.common.io.ByteStreams', + 'com.google.common.util.concurrent.ListenableFuture', + 'io.grpc.CallOptions', + 'io.grpc.Channel', + 'io.grpc.Drainable', + 'io.grpc.KnownLength', + 'io.grpc.ManagedChannel', + 'io.grpc.MethodDescriptor', + 'io.grpc.MethodDescriptor$Builder', + 'io.grpc.MethodDescriptor$Marshaller', + 'io.grpc.MethodDescriptor$MethodType', + 'io.grpc.stub.AbstractFutureStub', + 'io.grpc.stub.AbstractStub', + 'io.grpc.stub.ClientCalls', + 'org.bouncycastle.jsse.BCSSLParameters', + 'org.bouncycastle.jsse.BCSSLSocket', + 'org.conscrypt.Conscrypt', + 'org.conscrypt.Conscrypt$Version', + 'org.conscrypt.ConscryptHostnameVerifier', + 'org.openjsse.javax.net.ssl.SSLParameters', + 'org.openjsse.javax.net.ssl.SSLSocket', 'io.opentelemetry.api.events.EventEmitter', 'io.opentelemetry.api.events.EventEmitterBuilder', 'io.opentelemetry.api.events.EventEmitterProvider', @@ -68,7 +101,8 @@ thirdPartyAudit { 'io.opentelemetry.extension.incubator.metrics.LongGauge', 'io.opentelemetry.extension.incubator.metrics.LongGaugeAdviceConfigurer', 'io.opentelemetry.extension.incubator.metrics.LongHistogramAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.LongUpDownCounterAdviceConfigurer' + 'io.opentelemetry.extension.incubator.metrics.LongUpDownCounterAdviceConfigurer', + 'kotlin.io.path.PathsKt' ) } diff --git a/plugins/telemetry-otel/licenses/kotlin-stdlib-1.7.10.jar.sha1 b/plugins/telemetry-otel/licenses/kotlin-stdlib-1.7.10.jar.sha1 new file mode 100644 index 0000000000000..4d119fbf4df70 --- /dev/null +++ b/plugins/telemetry-otel/licenses/kotlin-stdlib-1.7.10.jar.sha1 @@ -0,0 +1 @@ +d2abf9e77736acc4450dc4a3f707fa2c10f5099d \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/kotlin-stdlib-LICENSE.txt b/plugins/telemetry-otel/licenses/kotlin-stdlib-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/kotlin-stdlib-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/kotlin-stdlib-NOTICE.txt b/plugins/telemetry-otel/licenses/kotlin-stdlib-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/okhttp-4.11.0.jar.sha1 b/plugins/telemetry-otel/licenses/okhttp-4.11.0.jar.sha1 new file mode 100644 index 0000000000000..1fc0db6615cb5 --- /dev/null +++ b/plugins/telemetry-otel/licenses/okhttp-4.11.0.jar.sha1 @@ -0,0 +1 @@ +436932d695b2c43f2c86b8111c596179cd133d56 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/okhttp-LICENSE.txt b/plugins/telemetry-otel/licenses/okhttp-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/okhttp-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/okhttp-NOTICE.txt b/plugins/telemetry-otel/licenses/okhttp-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/okio-jvm-3.5.0.jar.sha1 b/plugins/telemetry-otel/licenses/okio-jvm-3.5.0.jar.sha1 new file mode 100644 index 0000000000000..7b19d32d872fa --- /dev/null +++ b/plugins/telemetry-otel/licenses/okio-jvm-3.5.0.jar.sha1 @@ -0,0 +1 @@ +d6a0bc7343210eff7dd5cfdd6eb9b5f0036638ce \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/okio-jvm-LICENSE.txt b/plugins/telemetry-otel/licenses/okio-jvm-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/okio-jvm-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/okio-jvm-NOTICE.txt b/plugins/telemetry-otel/licenses/okio-jvm-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..eccb15f7b7c8e --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1 @@ -0,0 +1 @@ +f299d336dba1039478497f37b273dfa764c6faef \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..e88b7514ee54d --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1 @@ -0,0 +1 @@ +15692246539571c41180aff2b55abe527b939a7b \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..86937743208c6 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1 @@ -0,0 +1 @@ +947cf43a6411c4a323e14594431040a476ad43e8 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1 new file mode 100644 index 0000000000000..068926277253c --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1 @@ -0,0 +1 @@ +9f3a14515500e4df260ce7b10a668237a95ac791 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-LICENSE.txt new file mode 100644 index 0000000000000..6b0b1270ff0ca --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-LICENSE.txt @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java index b395a335a4d83..fe05cc8bb7a41 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java @@ -14,6 +14,8 @@ import org.opensearch.telemetry.tracing.sampler.ProbabilisticSampler; import org.opensearch.telemetry.tracing.sampler.RequestSampler; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.concurrent.TimeUnit; import io.opentelemetry.api.OpenTelemetry; @@ -45,11 +47,13 @@ private OTelResourceProvider() {} * @return OpenTelemetry instance */ public static OpenTelemetry get(TelemetrySettings telemetrySettings, Settings settings) { - return get( - settings, - OTelSpanExporterFactory.create(settings), - ContextPropagators.create(W3CTraceContextPropagator.getInstance()), - Sampler.parentBased(new RequestSampler(new ProbabilisticSampler(telemetrySettings))) + return AccessController.doPrivileged( + (PrivilegedAction) () -> get( + settings, + OTelSpanExporterFactory.create(settings), + ContextPropagators.create(W3CTraceContextPropagator.getInstance()), + Sampler.parentBased(new RequestSampler(new ProbabilisticSampler(telemetrySettings))) + ) ); } diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java index cd4ffb160903b..da7ce5c47d9ca 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactory.java @@ -16,6 +16,7 @@ import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; +import java.lang.reflect.Method; import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; @@ -55,9 +56,17 @@ private static SpanExporter instantiateSpanExporter(Class spanExpo // Check we ourselves are not being called by unprivileged code. SpecialPermission.check(); return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + String methodName = "create"; + String getDefaultMethod = "getDefault"; + for (Method m : spanExporterProviderClass.getMethods()) { + if (m.getName().equals(getDefaultMethod)) { + methodName = getDefaultMethod; + break; + } + } try { return (SpanExporter) MethodHandles.publicLookup() - .findStatic(spanExporterProviderClass, "create", MethodType.methodType(spanExporterProviderClass)) + .findStatic(spanExporterProviderClass, methodName, MethodType.methodType(spanExporterProviderClass)) .asType(MethodType.methodType(SpanExporter.class)) .invokeExact(); } catch (Throwable e) { diff --git a/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy index 4480cbb2bab4b..726db3d3f4700 100644 --- a/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy @@ -9,6 +9,8 @@ grant { permission java.lang.RuntimePermission "getClassLoader"; permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.net.NetPermission "getProxySelector"; + permission java.net.SocketPermission "*", "connect,resolve"; }; diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporterWithGetDefault.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporterWithGetDefault.java new file mode 100644 index 0000000000000..225cfa6ab2d1a --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/DummySpanExporterWithGetDefault.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.exporter; + +import java.util.Collection; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SpanExporter; + +public class DummySpanExporterWithGetDefault implements SpanExporter { + + public static DummySpanExporterWithGetDefault getDefault() { + return new DummySpanExporterWithGetDefault(); + } + + @Override + public CompletableResultCode export(Collection spans) { + return null; + } + + @Override + public CompletableResultCode flush() { + return null; + } + + @Override + public CompletableResultCode shutdown() { + return null; + } +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java index d9a86ccd3180b..d71aef9366e21 100644 --- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/exporter/OTelSpanExporterFactoryTests.java @@ -63,4 +63,15 @@ public void testSpanExporterNonSpanExporterClass() { } + public void testSpanExporterGetDefaultMethod() { + Settings settings = Settings.builder() + .put( + OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING.getKey(), + "org.opensearch.telemetry.tracing.exporter.DummySpanExporterWithGetDefault" + ) + .build(); + + assertTrue(OTelSpanExporterFactory.create(settings) instanceof DummySpanExporterWithGetDefault); + } + } From 05ee0c3f3f8b347371a9f77f89023d1ec9851707 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Thu, 21 Sep 2023 15:05:00 -0700 Subject: [PATCH 055/388] Removing extra protobuf notice file (#10114) * Removing the extra protobuf java notice from root Signed-off-by: Vacha Shah * Fixing precommit errors Signed-off-by: Vacha Shah --------- Signed-off-by: Vacha Shah --- .../core/index/mapper/CorrelationVectorFieldMapperTests.java | 2 +- protobuf-java-NOTICE.txt | 0 .../src/test/java/org/opensearch/upgrades/IndexingIT.java | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 protobuf-java-NOTICE.txt diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java index 70311a9279bbd..674f35069a742 100644 --- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java +++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapperTests.java @@ -191,7 +191,7 @@ public void testTypeParser_parse_fromCorrelationParamsContext_InvalidVectorSimil /** * test parseCreateField in CorrelationVectorFieldMapper - * @throws IOException + * @throws IOException ioexception */ public void testCorrelationVectorFieldMapper_parseCreateField() throws IOException { String fieldName = "test-field-name"; diff --git a/protobuf-java-NOTICE.txt b/protobuf-java-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index 418d6b0eda469..aef363058b394 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -259,7 +259,7 @@ public void testIndexing() throws Exception { * This test verifies that during rolling upgrades the segment replication does not break when replica shards can * be running on older codec versions. * - * @throws Exception + * @throws Exception exception */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7679") public void testIndexingWithSegRep() throws Exception { From fc1d51d567fb97838e2880003696b1884ad897bf Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Thu, 21 Sep 2023 18:27:02 -0400 Subject: [PATCH 056/388] Add search pipeline README (#9825) --------- Signed-off-by: Stephen Crawford --- modules/search-pipeline-common/README.md | 202 +++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 modules/search-pipeline-common/README.md diff --git a/modules/search-pipeline-common/README.md b/modules/search-pipeline-common/README.md new file mode 100644 index 0000000000000..70615d846987b --- /dev/null +++ b/modules/search-pipeline-common/README.md @@ -0,0 +1,202 @@ +- [Search Pipelines](#search-pipelines) + - [Architecture](#architecture) + - [Search Processors](#search-processors) + - [Creating a Search Processor](#creating-a-search-processor) + - [Creating a Pipeline](#creating-a-search-pipeline) + +# Search Pipelines + +This README briefly covers the two types of search processors, explains how you can use them to create search pipelines, and walks through the creation of a new processor. + +## Architecture + +Search pipelines allow cluster operators to create and reuse [components](#search-processors) to transform search queries and results. + +With search pipelines, the operator can combine multiple [search processors](#search-processors) to create a transform which acts on the search request and/or search response. + +Search pipelines offer numerous benefits: + +1. search processors living in OpenSearch can be used by _all_ calling applications; +2. search pipeline operations occur inside the OpenSearch cluster, so large results can be processed before returning to the calling application\*; +3. search processors can be distributed in plugins to be shared with other OpenSearch users; +4. search pipelines only need to be modified once (and without changing or redeploying any calling applications) to have a change occur on all incoming queries\*\*; +5. search pipelines support standard APIs for accessing metrics and disaster recovery. + +*Within a cluster, results are passed using a more efficient, but version-specific binary protocol. You can pass result information back to a coordinator, allow it to post-process (e.g. rerank or collapse), and finally truncate it before sending it to the client over the less efficient but flexible JSON API. + +**For example, the `FilterQueryRequestProcessor` could be used to exclude search results immediately, without needing to make a code change in the application layer and deploy the change across your fleet. + +## Search Processors + +You can create many search pipelines by combining search processors in various orders. There are two types of search processors: + +1. search request processors which transform a request _before_ it is executed; +2. search response processors which transform the output of a request _after_ it is executed. + +You can find all existing search processors registered in `SearchPipelineCommonModulePlugin.java` and described on the documentation website. + +### Creating a search processor + +New search processors can be created in two different ways. + +Generally, a search processor can be created in your own `SearchPipelinePlugin`. This method is best for when you are creating a unique search +processor for your niche application. This method should also be used when your processor relies on an outside service. To get started creating a search processor in a `SearchPipelinePlugin`, you can use the [plugin template](https://github.com/opensearch-project/opensearch-plugin-template-java ). + +Alternatively, if you think your processor may be valuable to _all_ OpenSearch users you can follow these steps: + +1. Create a new class in `org.opensearch.search.pipeline.common`, this class will hold your new processor and should include whether it is a request or response processor. For example, a response processor which deleted a target field could be called `DeleteFieldResponseProcessor`. + +2. Make the class extend the generic `AbstractProcessor` class as well as implement either the `SearchRequestProcessor` or `SearchResponseProcessor` class depending on what type of processor it is. In the `DeleteFieldResponseProcessor` example, this would look like: + +```public class DeleteFieldResponseProcessor extends AbstractProcessor implements SearchResponseProcessor``` + +3. Create the main functionality of your processor and implement the methods required by the implemented interface. This will be `SearchRequest processRequest(SearchRequest request) throws Exception;` for a search request processor or `SearchResponse processResponse(SearchRequest request, SearchResponse response) throws Exception;` for a search response processor. + +For the example field `DeleteFieldResponseProcessor`, this will look like: + +``` +@Override +public SearchResponse processResponse(SearchRequest request, SearchResponse response) throws Exception { + + boolean foundField = false; + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + + // Process each hit as desired + + if (hit.hasSource()) { + // Change hit source if needed + ); + + Map sourceAsMap = typeAndSourceMap.v2(); + if (sourceAsMap.containsKey(field)) { + // Handle source as map + } + } + + if (!foundField && !ignoreMissing) { + // Handle error scenarios + } + + return response; +} +``` + +4. Create a factory to parse processor-specific JSON configurations. These are used for constructing a processor instance. + +In the `DeleteFieldResponseProcessor`, this would look something like: + +``` +public static final class Factory implements Processor.Factory { + + /** + * Constructor for factory + */ + Factory() {} + + @Override + public DeleteFieldResponseProcessor create( + Map> processorFactories, + String tag, + String description, + boolean ignoreFailure, + Map config, + PipelineContext pipelineContext + ) throws Exception { + String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_missing", false); + return new DeleteFieldResponseProcessor(tag, description, ignoreFailure, field, ignoreMissing); + } +} +``` + +In this example, we provide specific configurations for which field should be deleted and whether the processor should ignore attempts to remove a non-existent field. + +5. Add the newly added search processor to the `SearchPieplineCommonModulePlugin` getter for the corresponding processor type. + +For the `DeleteFieldResponseProcessor`, you would modify the response processor getter to have: + +``` +@Override +public Map> getResponseProcessors(Parameters parameters) { + return Map.of( + RenameFieldResponseProcessor.TYPE, + new RenameFieldResponseProcessor.Factory(), + DeleteFieldResponseProcessor.TYPE, + new DeleteFieldResponseProcessor.Factory() + ); +} +``` + +6. After creating a search processor, the processor is ready to be tested in a search pipeline. + +To test your new search processor, you can make use of the test [`SearchPipelineCommonYamlTestSuiteIT`](src/yamlRestTest/java/org/opensearch/search/pipeline/common). + +Following the format of the YAML files in [`rest-api-spec.test.search_pipeline`](src/yamlRestTest/resources/rest-api-spec/test/search_pipeline), you should be able to create your own YAML test file to exercise your new processor. + +To run the tests, from the root of the OpenSearch repository, you can run `./gradlew :modules:search-pipeline-common:yamlRestTest`. + +7. Finally, the processor is ready to used in a cluster. + +To use the new processor, make sure the cluster is reloaded and that the new processor is accessible. + +The new processor should show when calling `GET /_nodes/search_pipelines`. + +If the new processor is shown in the cURL response, the new processor should be available for use in a search pipeline. + +## Creating a Search Pipeline + +To create a search pipeline, you must create an ordered list of search processors in the OpenSearch cluster. + +An example creation request is: + +``` +PUT /_search/pipeline/my_pipeline +{ + "request_processors": [ + { + "filter_query" : { + "tag" : "tag1", + "description" : "This processor is going to restrict to publicly visible documents", + "query" : { + "term": { + "visibility": "public" + } + } + } + } + ], + "response_processors": [ + { + "rename_field": { + "field": "message", + "target_field": "notification" + } + } + ] +} +``` + +Alternatively, if you want to use just the `DeleteFieldResponseProcessor` created before, you would use: + +``` +PUT /_search/pipeline/my_pipeline2 + +{ + "response_processors": [ + { + "delete_field": { + "field": "message" + } + } + ] +} +``` + +## Running a search request using a search pipeline + +To run a search request using a search pipeline, you first need to create the pipeline using the request format shown above. + +After that is completed, you can run a request using the format: `POST /myindex/_search?search_pipeline=`. + +In the example of the `DeleteFieldResponseProcessor` this would be called with `POST /myindex/_search?search_pipeline=my_pipeline2`. From c178d8ebadc8b9e7a409f5ddcda972e3c3baf6cb Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Thu, 21 Sep 2023 15:49:47 -0700 Subject: [PATCH 057/388] Fix NPE in EmptyTopDocsCollectorContext::createManager (#10082) Signed-off-by: Jay Deng --- CHANGELOG.md | 3 +- .../simple/ParameterizedSimpleSearchIT.java | 608 ++++++++++++++++++ .../search/simple/SimpleSearchIT.java | 534 +-------------- .../search/query/TopDocsCollectorContext.java | 4 +- 4 files changed, 614 insertions(+), 535 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c1108cdf3ee1..22cf61f77f886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,7 +109,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725)) -- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/9725)) +- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045)) +- Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082)) ### Security diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java new file mode 100644 index 0000000000000..719b75079da92 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java @@ -0,0 +1,608 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.simple; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchPhaseExecutionException; +import org.opensearch.action.search.SearchRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest.RefreshPolicy; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.rescore.QueryRescorerBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.index.query.QueryBuilders.boolQuery; +import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.index.query.QueryBuilders.queryStringQuery; +import static org.opensearch.index.query.QueryBuilders.rangeQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class ParameterizedSimpleSearchIT extends ParameterizedOpenSearchIntegTestCase { + + public ParameterizedSimpleSearchIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + + public void testSearchNullIndex() { + expectThrows( + NullPointerException.class, + () -> client().prepareSearch((String) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() + ); + + expectThrows( + NullPointerException.class, + () -> client().prepareSearch((String[]) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() + ); + + } + + public void testSearchRandomPreference() throws InterruptedException, ExecutionException { + createIndex("test"); + indexRandom( + true, + client().prepareIndex("test").setId("1").setSource("field", "value"), + client().prepareIndex("test").setId("2").setSource("field", "value"), + client().prepareIndex("test").setId("3").setSource("field", "value"), + client().prepareIndex("test").setId("4").setSource("field", "value"), + client().prepareIndex("test").setId("5").setSource("field", "value"), + client().prepareIndex("test").setId("6").setSource("field", "value") + ); + + int iters = scaledRandomIntBetween(10, 20); + for (int i = 0; i < iters; i++) { + String randomPreference = randomUnicodeOfLengthBetween(0, 4); + // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) + while (randomPreference.startsWith("_")) { + randomPreference = randomUnicodeOfLengthBetween(0, 4); + } + // id is not indexed, but lets see that we automatically convert to + SearchResponse searchResponse = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setPreference(randomPreference) + .get(); + assertHitCount(searchResponse, 6L); + + } + } + + public void testSimpleIp() throws Exception { + createIndex("test"); + + client().admin() + .indices() + .preparePutMapping("test") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject(MapperService.SINGLE_MAPPING_NAME) + .startObject("properties") + .startObject("from") + .field("type", "ip") + .endObject() + .startObject("to") + .field("type", "ip") + .endObject() + .endObject() + .endObject() + .endObject() + ) + .get(); + + client().prepareIndex("test").setId("1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefreshPolicy(IMMEDIATE).get(); + + SearchResponse search = client().prepareSearch() + .setQuery(boolQuery().must(rangeQuery("from").lte("192.168.0.7")).must(rangeQuery("to").gte("192.168.0.7"))) + .get(); + + assertHitCount(search, 1L); + } + + public void testIpCidr() throws Exception { + createIndex("test"); + + client().admin() + .indices() + .preparePutMapping("test") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject(MapperService.SINGLE_MAPPING_NAME) + .startObject("properties") + .startObject("ip") + .field("type", "ip") + .endObject() + .endObject() + .endObject() + .endObject() + ) + .get(); + ensureGreen(); + + client().prepareIndex("test").setId("1").setSource("ip", "192.168.0.1").get(); + client().prepareIndex("test").setId("2").setSource("ip", "192.168.0.2").get(); + client().prepareIndex("test").setId("3").setSource("ip", "192.168.0.3").get(); + client().prepareIndex("test").setId("4").setSource("ip", "192.168.1.4").get(); + client().prepareIndex("test").setId("5").setSource("ip", "2001:db8::ff00:42:8329").get(); + refresh(); + + SearchResponse search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(queryStringQuery("ip: 192.168.0.1")).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/32"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.0/24"))).get(); + assertHitCount(search, 3L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))).get(); + assertHitCount(search, 4L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))).get(); + assertHitCount(search, 4L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::ff00:42:8329/128"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::/64"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "::/0"))).get(); + assertHitCount(search, 5L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.5/32"))).get(); + assertHitCount(search, 0L); + + assertFailures( + client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0/0/0/0/0"))), + RestStatus.BAD_REQUEST, + containsString("Expected [ip/prefix] but was [0/0/0/0/0]") + ); + } + + public void testSimpleId() { + createIndex("test"); + + client().prepareIndex("test").setId("XXX1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + // id is not indexed, but lets see that we automatically convert to + SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")).get(); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")).get(); + assertHitCount(searchResponse, 1L); + } + + public void testSimpleDateRange() throws Exception { + createIndex("test"); + client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); + client().prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); + ensureGreen(); + refresh(); + SearchResponse searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lte("2010-01-06T02:00")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lt("2010-01-06T02:00")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00").lt("2010-01-06T02:00")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 0L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")) + .get(); + assertHitCount(searchResponse, 2L); + } + + // TODO: combine this test with SimpleSearchIT.testSimpleTerminateAfterCount after + // https://github.com/opensearch-project/OpenSearch/issues/8371 + public void testSimpleTerminateAfterCountWithSizeAndTrackHits() throws Exception { + prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); + ensureGreen(); + int numDocs = randomIntBetween(15, 29); + List docbuilders = new ArrayList<>(numDocs); + + for (int i = 1; i <= numDocs; i++) { + String id = String.valueOf(i); + docbuilders.add(client().prepareIndex("test").setId(id).setSource("field", i)); + } + + indexRandom(true, docbuilders); + ensureGreen(); + refresh(); + + SearchResponse searchResponse; + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.matchAllQuery()) + .setTerminateAfter(numDocs) + .setSize(0) + .setTrackTotalHits(true) + .get(); + assertEquals(0, searchResponse.getFailedShards()); + } + + public void testSimpleIndexSortEarlyTerminate() throws Exception { + prepareCreate("test").setSettings( + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).put("index.sort.field", "rank") + ).setMapping("rank", "type=integer").get(); + ensureGreen(); + int max = randomIntBetween(3, 29); + List docbuilders = new ArrayList<>(max); + + for (int i = max - 1; i >= 0; i--) { + String id = String.valueOf(i); + docbuilders.add(client().prepareIndex("test").setId(id).setSource("rank", i)); + } + + indexRandom(true, docbuilders); + ensureGreen(); + refresh(); + + SearchResponse searchResponse; + for (int i = 1; i < max; i++) { + searchResponse = client().prepareSearch("test") + .addDocValueField("rank") + .setTrackTotalHits(false) + .addSort("rank", SortOrder.ASC) + .setSize(i) + .get(); + assertNull(searchResponse.getHits().getTotalHits()); + for (int j = 0; j < i; j++) { + assertThat(searchResponse.getHits().getAt(j).field("rank").getValue(), equalTo((long) j)); + } + } + } + + public void testInsaneFromAndSize() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE)); + assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE)); + } + + public void testTooLargeFromAndSize() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))); + assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1)); + assertWindowFails( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + ); + } + + public void testLargeFromAndSizeSucceeds() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2 - 1) + .get(), + 1 + ); + } + + public void testTooLargeFromAndSizeOkBySetting() throws Exception { + prepareCreate("idx").setSettings( + Settings.builder() + .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2) + ).get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .get(), + 1 + ); + } + + public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { + createIndex("idx"); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("idx") + .setSettings( + Settings.builder() + .put( + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), + IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2 + ) + ) + .get() + ); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .get(), + 1 + ); + } + + public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception { + prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)).get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) + .get(), + 1 + ); + } + + public void testTooLargeRescoreWindow() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertRescoreWindowFails(Integer.MAX_VALUE); + assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1); + } + + public void testTooLargeRescoreOkBySetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) + .get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testTooLargeRescoreOkByResultWindowSetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + prepareCreate("idx").setSettings( + Settings.builder() + .put( + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window. + defaultMaxWindow * 2 + ) + ).get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testTooLargeRescoreOkByDynamicSetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + createIndex("idx"); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("idx") + .setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) + .get() + ); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + createIndex("idx"); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("idx") + .setSettings( + // Note that this is the RESULT window + Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2) + ) + .get() + ); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testQueryNumericFieldWithRegex() throws Exception { + assertAcked(prepareCreate("idx").setMapping("num", "type=integer")); + ensureGreen("idx"); + + try { + client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get(); + fail("SearchPhaseExecutionException should have been thrown"); + } catch (SearchPhaseExecutionException ex) { + assertThat(ex.getRootCause().getMessage(), containsString("Can only use regexp queries on keyword and text fields")); + } + } + + public void testTermQueryBigInt() throws Exception { + prepareCreate("idx").setMapping("field", "type=keyword").get(); + ensureGreen("idx"); + + client().prepareIndex("idx") + .setId("1") + .setSource("{\"field\" : 80315953321748200608 }", MediaTypeRegistry.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); + + String queryJson = "{ \"field\" : { \"value\" : 80315953321748200608 } }"; + XContentParser parser = createParser(JsonXContent.jsonXContent, queryJson); + parser.nextToken(); + TermQueryBuilder query = TermQueryBuilder.fromXContent(parser); + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(query).get(); + assertEquals(1, searchResponse.getHits().getTotalHits().value); + } + + public void testTooLongRegexInRegexpQuery() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + int defaultMaxRegexLength = IndexSettings.MAX_REGEX_LENGTH_SETTING.get(Settings.EMPTY); + StringBuilder regexp = new StringBuilder(defaultMaxRegexLength); + while (regexp.length() <= defaultMaxRegexLength) { + regexp.append("]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*))*(?:,@(?:(?:\\r\\n)?[ \\t])*(?:[^()<>@,;:\\\\\".\\"); + } + SearchPhaseExecutionException e = expectThrows( + SearchPhaseExecutionException.class, + () -> client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", regexp.toString())).get() + ); + assertThat( + e.getRootCause().getMessage(), + containsString( + "The length of regex [" + + regexp.length() + + "] used in the Regexp Query request has exceeded " + + "the allowed maximum of [" + + defaultMaxRegexLength + + "]. " + + "This maximum can be set by changing the [" + + IndexSettings.MAX_REGEX_LENGTH_SETTING.getKey() + + "] index level setting." + ) + ); + } + + private void assertWindowFails(SearchRequestBuilder search) { + SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); + assertThat( + e.toString(), + containsString( + "Result window is too large, from + size must be less than or equal to: [" + + IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + ) + ); + assertThat(e.toString(), containsString("See the scroll api for a more efficient way to request large data sets")); + } + + private void assertRescoreWindowFails(int windowSize) { + SearchRequestBuilder search = client().prepareSearch("idx") + .addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(windowSize)); + SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); + assertThat( + e.toString(), + containsString( + "Rescore window [" + + windowSize + + "] is too large. It must " + + "be less than [" + + IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + ) + ); + assertThat( + e.toString(), + containsString( + "This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting." + ) + ); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index 0e6073ad11689..67e460653245e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -6,258 +6,24 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.search.simple; import org.opensearch.action.index.IndexRequestBuilder; -import org.opensearch.action.search.SearchPhaseExecutionException; -import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.search.rescore.QueryRescorerBuilder; -import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; -import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.opensearch.index.query.QueryBuilders.boolQuery; -import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.index.query.QueryBuilders.queryStringQuery; -import static org.opensearch.index.query.QueryBuilders.rangeQuery; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; public class SimpleSearchIT extends OpenSearchIntegTestCase { - public void testSearchNullIndex() { - expectThrows( - NullPointerException.class, - () -> client().prepareSearch((String) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() - ); - - expectThrows( - NullPointerException.class, - () -> client().prepareSearch((String[]) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() - ); - - } - - public void testSearchRandomPreference() throws InterruptedException, ExecutionException { - createIndex("test"); - indexRandom( - true, - client().prepareIndex("test").setId("1").setSource("field", "value"), - client().prepareIndex("test").setId("2").setSource("field", "value"), - client().prepareIndex("test").setId("3").setSource("field", "value"), - client().prepareIndex("test").setId("4").setSource("field", "value"), - client().prepareIndex("test").setId("5").setSource("field", "value"), - client().prepareIndex("test").setId("6").setSource("field", "value") - ); - - int iters = scaledRandomIntBetween(10, 20); - for (int i = 0; i < iters; i++) { - String randomPreference = randomUnicodeOfLengthBetween(0, 4); - // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) - while (randomPreference.startsWith("_")) { - randomPreference = randomUnicodeOfLengthBetween(0, 4); - } - // id is not indexed, but lets see that we automatically convert to - SearchResponse searchResponse = client().prepareSearch() - .setQuery(QueryBuilders.matchAllQuery()) - .setPreference(randomPreference) - .get(); - assertHitCount(searchResponse, 6L); - - } - } - - public void testSimpleIp() throws Exception { - createIndex("test"); - - client().admin() - .indices() - .preparePutMapping("test") - .setSource( - XContentFactory.jsonBuilder() - .startObject() - .startObject(MapperService.SINGLE_MAPPING_NAME) - .startObject("properties") - .startObject("from") - .field("type", "ip") - .endObject() - .startObject("to") - .field("type", "ip") - .endObject() - .endObject() - .endObject() - .endObject() - ) - .get(); - - client().prepareIndex("test").setId("1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefreshPolicy(IMMEDIATE).get(); - - SearchResponse search = client().prepareSearch() - .setQuery(boolQuery().must(rangeQuery("from").lte("192.168.0.7")).must(rangeQuery("to").gte("192.168.0.7"))) - .get(); - - assertHitCount(search, 1L); - } - - public void testIpCidr() throws Exception { - createIndex("test"); - - client().admin() - .indices() - .preparePutMapping("test") - .setSource( - XContentFactory.jsonBuilder() - .startObject() - .startObject(MapperService.SINGLE_MAPPING_NAME) - .startObject("properties") - .startObject("ip") - .field("type", "ip") - .endObject() - .endObject() - .endObject() - .endObject() - ) - .get(); - ensureGreen(); - - client().prepareIndex("test").setId("1").setSource("ip", "192.168.0.1").get(); - client().prepareIndex("test").setId("2").setSource("ip", "192.168.0.2").get(); - client().prepareIndex("test").setId("3").setSource("ip", "192.168.0.3").get(); - client().prepareIndex("test").setId("4").setSource("ip", "192.168.1.4").get(); - client().prepareIndex("test").setId("5").setSource("ip", "2001:db8::ff00:42:8329").get(); - refresh(); - - SearchResponse search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(queryStringQuery("ip: 192.168.0.1")).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/32"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.0/24"))).get(); - assertHitCount(search, 3L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))).get(); - assertHitCount(search, 4L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))).get(); - assertHitCount(search, 4L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::ff00:42:8329/128"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::/64"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "::/0"))).get(); - assertHitCount(search, 5L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.5/32"))).get(); - assertHitCount(search, 0L); - - assertFailures( - client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0/0/0/0/0"))), - RestStatus.BAD_REQUEST, - containsString("Expected [ip/prefix] but was [0/0/0/0/0]") - ); - } - - public void testSimpleId() { - createIndex("test"); - - client().prepareIndex("test").setId("XXX1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); - // id is not indexed, but lets see that we automatically convert to - SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")).get(); - assertHitCount(searchResponse, 1L); - - searchResponse = client().prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")).get(); - assertHitCount(searchResponse, 1L); - } - - public void testSimpleDateRange() throws Exception { - createIndex("test"); - client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); - client().prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); - ensureGreen(); - refresh(); - SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 2L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lte("2010-01-06T02:00")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 2L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lt("2010-01-06T02:00")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 1L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00").lt("2010-01-06T02:00")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 0L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")) - .get(); - assertHitCount(searchResponse, 2L); - } - + // TODO: Move this test to ParameterizedSimpleSearchIT after https://github.com/opensearch-project/OpenSearch/issues/8371 public void testSimpleTerminateAfterCount() throws Exception { prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); ensureGreen(); @@ -291,302 +57,4 @@ public void testSimpleTerminateAfterCount() throws Exception { assertHitCount(searchResponse, max); assertFalse(searchResponse.isTerminatedEarly()); } - - public void testSimpleIndexSortEarlyTerminate() throws Exception { - prepareCreate("test").setSettings( - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).put("index.sort.field", "rank") - ).setMapping("rank", "type=integer").get(); - ensureGreen(); - int max = randomIntBetween(3, 29); - List docbuilders = new ArrayList<>(max); - - for (int i = max - 1; i >= 0; i--) { - String id = String.valueOf(i); - docbuilders.add(client().prepareIndex("test").setId(id).setSource("rank", i)); - } - - indexRandom(true, docbuilders); - ensureGreen(); - refresh(); - - SearchResponse searchResponse; - for (int i = 1; i < max; i++) { - searchResponse = client().prepareSearch("test") - .addDocValueField("rank") - .setTrackTotalHits(false) - .addSort("rank", SortOrder.ASC) - .setSize(i) - .get(); - assertNull(searchResponse.getHits().getTotalHits()); - for (int j = 0; j < i; j++) { - assertThat(searchResponse.getHits().getAt(j).field("rank").getValue(), equalTo((long) j)); - } - } - } - - public void testInsaneFromAndSize() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE)); - assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE)); - } - - public void testTooLargeFromAndSize() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))); - assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1)); - assertWindowFails( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - ); - } - - public void testLargeFromAndSizeSucceeds() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2 - 1) - .get(), - 1 - ); - } - - public void testTooLargeFromAndSizeOkBySetting() throws Exception { - prepareCreate("idx").setSettings( - Settings.builder() - .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2) - ).get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .get(), - 1 - ); - } - - public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { - createIndex("idx"); - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings("idx") - .setSettings( - Settings.builder() - .put( - IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), - IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2 - ) - ) - .get() - ); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .get(), - 1 - ); - } - - public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception { - prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)).get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) - .get(), - 1 - ); - } - - public void testTooLargeRescoreWindow() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertRescoreWindowFails(Integer.MAX_VALUE); - assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1); - } - - public void testTooLargeRescoreOkBySetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) - .get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testTooLargeRescoreOkByResultWindowSetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - prepareCreate("idx").setSettings( - Settings.builder() - .put( - IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window. - defaultMaxWindow * 2 - ) - ).get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testTooLargeRescoreOkByDynamicSetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - createIndex("idx"); - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings("idx") - .setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) - .get() - ); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - createIndex("idx"); - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings("idx") - .setSettings( - // Note that this is the RESULT window - Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2) - ) - .get() - ); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testQueryNumericFieldWithRegex() throws Exception { - assertAcked(prepareCreate("idx").setMapping("num", "type=integer")); - ensureGreen("idx"); - - try { - client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get(); - fail("SearchPhaseExecutionException should have been thrown"); - } catch (SearchPhaseExecutionException ex) { - assertThat(ex.getRootCause().getMessage(), containsString("Can only use regexp queries on keyword and text fields")); - } - } - - public void testTermQueryBigInt() throws Exception { - prepareCreate("idx").setMapping("field", "type=keyword").get(); - ensureGreen("idx"); - - client().prepareIndex("idx") - .setId("1") - .setSource("{\"field\" : 80315953321748200608 }", MediaTypeRegistry.JSON) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); - - String queryJson = "{ \"field\" : { \"value\" : 80315953321748200608 } }"; - XContentParser parser = createParser(JsonXContent.jsonXContent, queryJson); - parser.nextToken(); - TermQueryBuilder query = TermQueryBuilder.fromXContent(parser); - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(query).get(); - assertEquals(1, searchResponse.getHits().getTotalHits().value); - } - - public void testTooLongRegexInRegexpQuery() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - int defaultMaxRegexLength = IndexSettings.MAX_REGEX_LENGTH_SETTING.get(Settings.EMPTY); - StringBuilder regexp = new StringBuilder(defaultMaxRegexLength); - while (regexp.length() <= defaultMaxRegexLength) { - regexp.append("]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*))*(?:,@(?:(?:\\r\\n)?[ \\t])*(?:[^()<>@,;:\\\\\".\\"); - } - SearchPhaseExecutionException e = expectThrows( - SearchPhaseExecutionException.class, - () -> client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", regexp.toString())).get() - ); - assertThat( - e.getRootCause().getMessage(), - containsString( - "The length of regex [" - + regexp.length() - + "] used in the Regexp Query request has exceeded " - + "the allowed maximum of [" - + defaultMaxRegexLength - + "]. " - + "This maximum can be set by changing the [" - + IndexSettings.MAX_REGEX_LENGTH_SETTING.getKey() - + "] index level setting." - ) - ); - } - - private void assertWindowFails(SearchRequestBuilder search) { - SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); - assertThat( - e.toString(), - containsString( - "Result window is too large, from + size must be less than or equal to: [" - + IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - ) - ); - assertThat(e.toString(), containsString("See the scroll api for a more efficient way to request large data sets")); - } - - private void assertRescoreWindowFails(int windowSize) { - SearchRequestBuilder search = client().prepareSearch("idx") - .addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(windowSize)); - SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); - assertThat( - e.toString(), - containsString( - "Rescore window [" - + windowSize - + "] is too large. It must " - + "be less than [" - + IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) - ) - ); - assertThat( - e.toString(), - containsString( - "This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting." - ) - ); - } } diff --git a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java index 39c34f7c0d5d5..86f6e542f97d1 100644 --- a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java @@ -183,7 +183,9 @@ CollectorManager createManager(CollectorManager( new TotalHitCountCollectorManager(sort), trackTotalHitsUpTo, From 130304054a3c3f2b15bf0ed96b2950c9a619572b Mon Sep 17 00:00:00 2001 From: Poojita Raj Date: Thu, 21 Sep 2023 17:19:42 -0700 Subject: [PATCH 058/388] Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 (#10126) --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- .../licenses/google-http-client-jackson2-1.43.2.jar.sha1 | 1 - .../licenses/google-http-client-jackson2-1.43.3.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.3.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 22cf61f77f886..36d566805ebf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098)) - Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125)) - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752)) +- Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 1f73d6a4d7520..81b55c77941e8 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -78,7 +78,7 @@ dependencies { api 'com.google.http-client:google-http-client:1.43.2' api 'com.google.http-client:google-http-client-appengine:1.43.2' api 'com.google.http-client:google-http-client-gson:1.43.3' - api 'com.google.http-client:google-http-client-jackson2:1.43.2' + api 'com.google.http-client:google-http-client-jackson2:1.43.3' api 'com.google.oauth-client:google-oauth-client:1.34.1' diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 deleted file mode 100644 index 7b606a07651ed..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e52a9967ebd8246fc4cca64df5f03608db5ac6e \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.3.jar.sha1 new file mode 100644 index 0000000000000..8380b9fb770b5 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.3.jar.sha1 @@ -0,0 +1 @@ +689da86469d19a01c726c8c24477b95c8a834bbe \ No newline at end of file From cbff21d7ea58d3185b969df43f48de10e32180e2 Mon Sep 17 00:00:00 2001 From: Ashish Date: Fri, 22 Sep 2023 08:25:58 +0530 Subject: [PATCH 059/388] Downgrade write lock to read lock before translog upload to remote store (#10135) * Revert "Optimize read write lock constructs during translog upload to remote store (#9636)" This reverts commit b5cc00222816429ed673d9fb46c434cae1be862b. Signed-off-by: Ashish Singh * Downgrade writelock to readlock during translog upload Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- .../index/translog/RemoteFsTranslog.java | 101 +++++++----------- .../TranslogCheckpointTransferSnapshot.java | 1 + .../transfer/TranslogTransferManager.java | 2 +- .../listener/TranslogTransferListener.java | 2 +- .../index/translog/RemoteFsTranslogTests.java | 1 + .../TranslogTransferManagerTests.java | 3 - 6 files changed, 40 insertions(+), 70 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index a633cb0787f27..23903e5a9d110 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.SetOnce; import org.opensearch.common.lease.Releasable; -import org.opensearch.common.lease.Releasables; import org.opensearch.common.logging.Loggers; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; @@ -60,7 +59,6 @@ public class RemoteFsTranslog extends Translog { private final BooleanSupplier primaryModeSupplier; private final RemoteTranslogTransferTracker remoteTranslogTransferTracker; private volatile long maxRemoteTranslogGenerationUploaded; - private final Object uploadMutex = new Object(); private volatile long minSeqNoToKeep; @@ -239,19 +237,10 @@ public static TranslogTransferManager buildTranslogTransferManager( @Override public boolean ensureSynced(Location location) throws IOException { try { - boolean shouldUpload = false; - try (ReleasableLock ignored = writeLock.acquire()) { - assert location.generation <= current.getGeneration(); - if (location.generation == current.getGeneration()) { - ensureOpen(); - if (prepareForUpload(location.generation) == false) { - return false; - } - shouldUpload = true; - } - } - if (shouldUpload) { - return performUpload(primaryTermSupplier.getAsLong(), location.generation); + assert location.generation <= current.getGeneration(); + if (location.generation == current.getGeneration()) { + ensureOpen(); + return prepareAndUpload(primaryTermSupplier.getAsLong(), location.generation); } } catch (final Exception ex) { closeOnTragicEvent(ex); @@ -266,12 +255,10 @@ public void rollGeneration() throws IOException { if (current.totalOperations() == 0 && primaryTermSupplier.getAsLong() == current.getPrimaryTerm()) { return; } - if (prepareForUpload(null)) { - performUpload(primaryTermSupplier.getAsLong(), null); - } + prepareAndUpload(primaryTermSupplier.getAsLong(), null); } - private boolean prepareForUpload(Long generation) throws IOException { + private boolean prepareAndUpload(Long primaryTerm, Long generation) throws IOException { try (Releasable ignored = writeLock.acquire()) { if (generation == null || generation == current.getGeneration()) { try { @@ -282,46 +269,36 @@ private boolean prepareForUpload(Long generation) throws IOException { logger.trace("Creating new writer for gen: [{}]", current.getGeneration() + 1); current = createWriter(current.getGeneration() + 1); } + assert writeLock.isHeldByCurrentThread() : "Write lock must be held before we acquire the read lock"; + // Here we are downgrading the write lock by acquiring the read lock and releasing the write lock + // This ensures that other threads can still acquire the read locks while also protecting the + // readers and writer to not be mutated any further. + readLock.acquire(); } catch (final Exception e) { tragedy.setTragicException(e); closeOnTragicEvent(e); throw e; } - return true; - } else return generation >= current.getGeneration(); + } else if (generation < current.getGeneration()) { + return false; + } } - } - /** - * This method does the remote store upload by first acquiring the lock on the uploadMutex monitor. The synchronized - * is required to restrict multiple uploads happening concurrently. The read lock is required to ensure that the - * underlying translog readers are not deleted and the current writer is not converted to a reader at the time of - * upload. - * - * @param primaryTerm current primary term - * @param generation current generation - * @return true if upload is successful - * @throws IOException if the upload fails due to any underlying exceptions. - */ - private boolean performUpload(Long primaryTerm, Long generation) throws IOException { - synchronized (uploadMutex) { - try (Releasable ignored = readLock.acquire()) { - // Do we need remote writes in sync fashion ? - // If we don't , we should swallow FileAlreadyExistsException while writing to remote store - // and also verify for same during primary-primary relocation - // Writing remote in sync fashion doesn't hurt as global ckp update - // is not updated in remote translog except in primary to primary recovery. - long generationToUpload; - if (generation == null) { - if (closed.get() == false) { - generationToUpload = current.getGeneration() - 1; - } else { - generationToUpload = current.getGeneration(); - } + assert readLock.isHeldByCurrentThread() == true; + try (Releasable ignored = readLock; Releasable ignoredGenLock = deletionPolicy.acquireTranslogGen(getMinFileGeneration())) { + // Do we need remote writes in sync fashion ? + // If we don't , we should swallow FileAlreadyExistsException while writing to remote store + // and also verify for same during primary-primary relocation + // Writing remote in sync fashion doesn't hurt as global ckp update + // is not updated in remote translog except in primary to primary recovery. + if (generation == null) { + if (closed.get() == false) { + return upload(primaryTerm, current.getGeneration() - 1); } else { - generationToUpload = generation; + return upload(primaryTerm, current.getGeneration()); } - return upload(primaryTerm, generationToUpload); + } else { + return upload(primaryTerm, generation); } } } @@ -347,10 +324,9 @@ private boolean upload(Long primaryTerm, Long generation) throws IOException { Translog::getCommitCheckpointFileName ).build() ) { - Releasable transferReleasable = Releasables.wrap(deletionPolicy.acquireTranslogGen(getMinFileGeneration())); return translogTransferManager.transferSnapshot( transferSnapshotProvider, - new RemoteFsTranslogTransferListener(transferReleasable, generation, primaryTerm) + new RemoteFsTranslogTransferListener(generation, primaryTerm) ); } @@ -373,8 +349,8 @@ private boolean syncToDisk() throws IOException { @Override public void sync() throws IOException { try { - if ((syncToDisk() || syncNeeded()) && prepareForUpload(null)) { - performUpload(primaryTermSupplier.getAsLong(), null); + if (syncToDisk() || syncNeeded()) { + prepareAndUpload(primaryTermSupplier.getAsLong(), null); } } catch (final Exception e) { tragedy.setTragicException(e); @@ -529,16 +505,17 @@ protected void onDelete() { translogTransferManager.delete(); } + // Visible for testing + boolean isRemoteGenerationDeletionPermitsAvailable() { + return remoteGenerationDeletionPermits.availablePermits() == REMOTE_DELETION_PERMITS; + } + /** * TranslogTransferListener implementation for RemoteFsTranslog * * @opensearch.internal */ private class RemoteFsTranslogTransferListener implements TranslogTransferListener { - /** - * Releasable instance for the translog - */ - private final Releasable transferReleasable; /** * Generation for the translog @@ -550,8 +527,7 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen */ private final Long primaryTerm; - RemoteFsTranslogTransferListener(Releasable transferReleasable, Long generation, Long primaryTerm) { - this.transferReleasable = transferReleasable; + RemoteFsTranslogTransferListener(Long generation, Long primaryTerm) { this.generation = generation; this.primaryTerm = primaryTerm; } @@ -571,10 +547,5 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) thro throw (RuntimeException) ex; } } - - @Override - public void close() { - transferReleasable.close(); - } } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java index eb0eebb564b63..10dec13c81e1a 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java @@ -164,6 +164,7 @@ public TranslogCheckpointTransferSnapshot build() throws IOException { translogTransferSnapshot.setMinTranslogGeneration(highestGenMinTranslogGeneration); assert this.primaryTerm == highestGenPrimaryTerm : "inconsistent primary term"; + assert this.generation == highestGeneration : " inconsistent generation "; final long finalHighestGeneration = highestGeneration; assert LongStream.iterate(lowestGeneration, i -> i + 1) .limit(highestGeneration) diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index fe6b5dab9937b..fd4936603671c 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -109,7 +109,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans long prevUploadBytesSucceeded = remoteTranslogTransferTracker.getUploadBytesSucceeded(); long prevUploadTimeInMillis = remoteTranslogTransferTracker.getTotalUploadTimeInMillis(); - try (translogTransferListener) { + try { toUpload.addAll(fileTransferTracker.exclusionFilter(transferSnapshot.getTranslogFileSnapshots())); toUpload.addAll(fileTransferTracker.exclusionFilter((transferSnapshot.getCheckpointFileSnapshots()))); if (toUpload.isEmpty()) { diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java b/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java index 8805c16298d96..132d1adf916da 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/listener/TranslogTransferListener.java @@ -17,7 +17,7 @@ * * @opensearch.internal */ -public interface TranslogTransferListener extends AutoCloseable { +public interface TranslogTransferListener { /** * Invoked when the transfer of {@link TransferSnapshot} succeeds * @param transferSnapshot the transfer snapshot diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 3c654818ffc6c..233d6f319b797 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -620,6 +620,7 @@ public void testSimpleOperationsUpload() throws Exception { translog.setMinSeqNoToKeep(2); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); translog.trimUnreferencedReaders(); assertEquals(1, translog.readers.size()); assertEquals(1, translog.stats().estimatedNumberOfOperations()); diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 0987201eb8602..6fc4557a75675 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -168,9 +168,6 @@ public void onUploadComplete(TransferSnapshot transferSnapshot) { public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) { translogTransferFailed.incrementAndGet(); } - - @Override - public void close() {} })); assertEquals(4, fileTransferSucceeded.get()); assertEquals(0, fileTransferFailed.get()); From 994e11518ca0754ed6275ae7f88744b913255323 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:21:11 -0400 Subject: [PATCH 060/388] Provide service accounts tokens to extensions (#9618) Provide service accounts tokens to extensions This change adds a new transport action which passes the extension a string representation of its service account auth token. This token is created by the TokenManager interface implementation. The token is expected to be an encoded basic auth credential string which can be used by the extension to interact with its own system index. Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Signed-off-by: Peter Nied Co-authored-by: Owais Kazi Co-authored-by: Peter Nied --- CHANGELOG.md | 1 + .../identity/shiro/ShiroTokenManager.java | 19 ++++++++-- .../identity/shiro/AuthTokenHandlerTests.java | 12 +++++++ .../discovery/InitializeExtensionRequest.java | 14 ++++++-- .../extensions/ExtensionsManager.java | 16 +++++++-- .../extensions/NoopExtensionsManager.java | 3 +- .../identity/noop/NoopTokenManager.java | 14 ++++++++ .../identity/tokens/BasicAuthToken.java | 2 +- .../identity/tokens/TokenManager.java | 8 +++++ .../main/java/org/opensearch/node/Node.java | 2 +- .../opensearch/action/ActionModuleTests.java | 2 +- .../InitializeExtensionRequestTests.java | 9 ++++- .../extensions/ExtensionsManagerTests.java | 36 +++++++++---------- .../RestInitializeExtensionActionTests.java | 13 +++++-- .../rest/RestSendToExtensionActionTests.java | 2 +- 15 files changed, 119 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36d566805ebf4..85211cff7f1bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107)) - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679)) - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110)) +- Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index 60239d609a08a..ddfb99e626718 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -64,9 +64,9 @@ public Optional translateAuthToken(org.opensearch.identity. public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) { String password = generatePassword(); - final byte[] rawEncoded = Base64.getEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); // Make a new - // ShiroSubject w/ - // audience as name + // Make a new ShiroSubject audience as name + final byte[] rawEncoded = Base64.getUrlEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); + final String usernamePassword = new String(rawEncoded, UTF_8); final String header = "Basic " + usernamePassword; BasicAuthToken token = new BasicAuthToken(header); @@ -75,6 +75,19 @@ public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) return token; } + @Override + public AuthToken issueServiceAccountToken(String audience) { + + String password = generatePassword(); + final byte[] rawEncoded = Base64.getUrlEncoder().withoutPadding().encode((audience + ":" + password).getBytes(UTF_8)); // Make a new + final String usernamePassword = new String(rawEncoded, UTF_8); + final String header = "Basic " + usernamePassword; + + BasicAuthToken token = new BasicAuthToken(header); + shiroTokenPasswordMap.put(token, password); + return token; + } + @Override public Subject authenticateToken(AuthToken authToken) { return new NoopSubject(); diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java index 7c7414843d86f..db77ced298991 100644 --- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java @@ -167,6 +167,18 @@ public void testTokenNoopIssuance() { Subject subject = new NoopSubject(); AuthToken token = tokenManager.issueOnBehalfOfToken(subject, claims); assertTrue(token instanceof AuthToken); + AuthToken serviceAccountToken = tokenManager.issueServiceAccountToken("test"); + assertTrue(serviceAccountToken instanceof AuthToken); + assertEquals(serviceAccountToken.asAuthHeaderValue(), "noopToken"); } + public void testShouldSucceedIssueServiceAccountToken() { + String audience = "testExtensionName"; + BasicAuthToken authToken = (BasicAuthToken) shiroAuthTokenHandler.issueServiceAccountToken(audience); + assertTrue(authToken instanceof BasicAuthToken); + UsernamePasswordToken translatedToken = (UsernamePasswordToken) shiroAuthTokenHandler.translateAuthToken(authToken).get(); + assertEquals(authToken.getPassword(), new String(translatedToken.getPassword())); + assertTrue(shiroAuthTokenHandler.getShiroTokenPasswordMap().containsKey(authToken)); + assertEquals(shiroAuthTokenHandler.getShiroTokenPasswordMap().get(authToken), new String(translatedToken.getPassword())); + } } diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java index 33cdad3045780..6e9fb8b7201a4 100644 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java @@ -25,16 +25,19 @@ public class InitializeExtensionRequest extends TransportRequest { private final DiscoveryNode sourceNode; private final DiscoveryExtensionNode extension; + private final String serviceAccountHeader; - public InitializeExtensionRequest(DiscoveryNode sourceNode, DiscoveryExtensionNode extension) { + public InitializeExtensionRequest(DiscoveryNode sourceNode, DiscoveryExtensionNode extension, String serviceAccountHeader) { this.sourceNode = sourceNode; this.extension = extension; + this.serviceAccountHeader = serviceAccountHeader; } public InitializeExtensionRequest(StreamInput in) throws IOException { super(in); sourceNode = new DiscoveryNode(in); extension = new DiscoveryExtensionNode(in); + serviceAccountHeader = in.readString(); } @Override @@ -42,6 +45,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); sourceNode.writeTo(out); extension.writeTo(out); + out.writeString(serviceAccountHeader); } public DiscoveryNode getSourceNode() { @@ -52,6 +56,10 @@ public DiscoveryExtensionNode getExtension() { return extension; } + public String getServiceAccountHeader() { + return serviceAccountHeader; + } + @Override public String toString() { return "InitializeExtensionsRequest{" + "sourceNode=" + sourceNode + ", extension=" + extension + '}'; @@ -62,7 +70,9 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; InitializeExtensionRequest that = (InitializeExtensionRequest) o; - return Objects.equals(sourceNode, that.sourceNode) && Objects.equals(extension, that.extension); + return Objects.equals(sourceNode, that.sourceNode) + && Objects.equals(extension, that.extension) + && Objects.equals(serviceAccountHeader, that.getServiceAccountHeader()); } @Override diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 61a6e2faa09e0..9f9ba548143c6 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -41,6 +41,7 @@ import org.opensearch.extensions.settings.CustomSettingsRequestHandler; import org.opensearch.extensions.settings.RegisterCustomSettingsRequest; import org.opensearch.identity.IdentityService; +import org.opensearch.identity.tokens.AuthToken; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.TransportException; @@ -101,6 +102,7 @@ public static enum OpenSearchRequestType { private Settings environmentSettings; private AddSettingsUpdateConsumerRequestHandler addSettingsUpdateConsumerRequestHandler; private NodeClient client; + private IdentityService identityService; /** * Instantiate a new ExtensionsManager object to handle requests and responses from extensions. This is called during Node bootstrap. @@ -108,7 +110,7 @@ public static enum OpenSearchRequestType { * @param additionalSettings Additional settings to read in from extension initialization request * @throws IOException If the extensions discovery file is not properly retrieved. */ - public ExtensionsManager(Set> additionalSettings) throws IOException { + public ExtensionsManager(Set> additionalSettings, IdentityService identityService) throws IOException { logger.info("ExtensionsManager initialized"); this.initializedExtensions = new HashMap(); this.extensionIdMap = new HashMap(); @@ -123,6 +125,7 @@ public ExtensionsManager(Set> additionalSettings) throws IOException } this.client = null; this.extensionTransportActionsHandler = null; + this.identityService = identityService; } /** @@ -400,7 +403,7 @@ protected void doRun() throws Exception { transportService.sendRequest( extension, REQUEST_EXTENSION_ACTION_NAME, - new InitializeExtensionRequest(transportService.getLocalNode(), extension), + new InitializeExtensionRequest(transportService.getLocalNode(), extension, issueServiceAccount(extension)), initializeExtensionResponseHandler ); } @@ -443,6 +446,15 @@ TransportResponse handleExtensionRequest(ExtensionRequest extensionRequest) thro } } + /** + * A helper method called during initialization that issues a service accounts to extensions + * @param extension The extension to be issued a service account + */ + private String issueServiceAccount(DiscoveryExtensionNode extension) { + AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getId()); + return serviceAccountToken.asAuthHeaderValue(); + } + static String getRequestExtensionActionName() { return REQUEST_EXTENSION_ACTION_NAME; } diff --git a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java index 34aa3ec1367a7..81b1b91b11481 100644 --- a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java @@ -20,6 +20,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -31,7 +32,7 @@ public class NoopExtensionsManager extends ExtensionsManager { public NoopExtensionsManager() throws IOException { - super(Set.of()); + super(Set.of(), new IdentityService(Settings.EMPTY, List.of())); } @Override diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java index 1255e822cea6e..1dc3a58916b5c 100644 --- a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java +++ b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java @@ -37,6 +37,20 @@ public String asAuthHeaderValue() { }; } + /** + * Issue a new Noop Token + * @return a new Noop Token + */ + @Override + public AuthToken issueServiceAccountToken(final String audience) { + return new AuthToken() { + @Override + public String asAuthHeaderValue() { + return "noopToken"; + } + }; + } + @Override public Subject authenticateToken(AuthToken authToken) { return null; diff --git a/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java b/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java index 71b8fe504a5d1..4ad0bbe67d2a1 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java +++ b/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java @@ -23,7 +23,7 @@ public final class BasicAuthToken implements AuthToken { public BasicAuthToken(final String headerValue) { final String base64Encoded = headerValue.substring(TOKEN_IDENTIFIER.length()).trim(); - final byte[] rawDecoded = Base64.getDecoder().decode(base64Encoded); + final byte[] rawDecoded = Base64.getUrlDecoder().decode(base64Encoded); final String usernamepassword = new String(rawDecoded, StandardCharsets.UTF_8); final String[] tokenParts = usernamepassword.split(":", 2); diff --git a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java index 4f6ddeb48dea3..b4048251a06a2 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java +++ b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java @@ -23,6 +23,14 @@ public interface TokenManager { */ public AuthToken issueOnBehalfOfToken(final Subject subject, final OnBehalfOfClaims claims); + /** + * Create a new service account token + * + * @param audience: A string representing the unique id of the extension for which a service account token should be generated + * @return a new auth token + */ + public AuthToken issueServiceAccountToken(final String audience); + /** * Authenticates a provided authToken * @param authToken: The authToken to authenticate diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 90fb339951d62..729b38ca27394 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -495,7 +495,7 @@ protected Node( for (ExtensionAwarePlugin extAwarePlugin : extensionAwarePlugins) { additionalSettings.addAll(extAwarePlugin.getExtensionSettings()); } - this.extensionsManager = new ExtensionsManager(additionalSettings); + this.extensionsManager = new ExtensionsManager(additionalSettings, identityService); } else { this.extensionsManager = new NoopExtensionsManager(); } diff --git a/server/src/test/java/org/opensearch/action/ActionModuleTests.java b/server/src/test/java/org/opensearch/action/ActionModuleTests.java index deb0bb58a5c4d..8479f011adf48 100644 --- a/server/src/test/java/org/opensearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/opensearch/action/ActionModuleTests.java @@ -143,7 +143,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() throws IOException { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of()) + new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) ); actionModule.initRestHandlers(null); // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail diff --git a/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java b/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java index ca94a3b52a56c..daa42d3abbc50 100644 --- a/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java +++ b/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java @@ -27,6 +27,7 @@ public class InitializeExtensionRequestTests extends OpenSearchTestCase { public void testInitializeExtensionRequest() throws Exception { String expectedUniqueId = "test uniqueid"; Version expectedVersion = Version.fromString("2.0.0"); + String expectedServiceAccountHeader = "test"; ExtensionDependency expectedDependency = new ExtensionDependency(expectedUniqueId, expectedVersion); DiscoveryExtensionNode expectedExtensionNode = new DiscoveryExtensionNode( "firstExtension", @@ -46,9 +47,14 @@ public void testInitializeExtensionRequest() throws Exception { Version.CURRENT ); - InitializeExtensionRequest initializeExtensionRequest = new InitializeExtensionRequest(expectedSourceNode, expectedExtensionNode); + InitializeExtensionRequest initializeExtensionRequest = new InitializeExtensionRequest( + expectedSourceNode, + expectedExtensionNode, + expectedServiceAccountHeader + ); assertEquals(expectedExtensionNode, initializeExtensionRequest.getExtension()); assertEquals(expectedSourceNode, initializeExtensionRequest.getSourceNode()); + assertEquals(expectedServiceAccountHeader, initializeExtensionRequest.getServiceAccountHeader()); try (BytesStreamOutput out = new BytesStreamOutput()) { initializeExtensionRequest.writeTo(out); @@ -58,6 +64,7 @@ public void testInitializeExtensionRequest() throws Exception { assertEquals(expectedExtensionNode, initializeExtensionRequest.getExtension()); assertEquals(expectedSourceNode, initializeExtensionRequest.getSourceNode()); + assertEquals(expectedServiceAccountHeader, initializeExtensionRequest.getServiceAccountHeader()); } } } diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 57883f1e5914a..f243a924f4e63 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -184,7 +184,7 @@ public void testLoadExtensions() throws Exception { Set> additionalSettings = extAwarePlugin.getExtensionSettings().stream().collect(Collectors.toSet()); ExtensionScopedSettings extensionScopedSettings = new ExtensionScopedSettings(additionalSettings); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); ExtensionDependency dependentExtension = new ExtensionDependency("uniqueid0", Version.fromString("2.0.0")); Extension firstExtension = new Extension( @@ -280,7 +280,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { null, null ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); extensionsManager.loadExtension(firstExtension); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(secondExtension)); assertEquals( @@ -319,7 +319,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { public void testMissingRequiredFieldsWhileLoadingExtension() throws Exception { Extension firstExtension = new Extension("firstExtension", "uniqueid1", "127.0.0.0", "9300", "0.0.7", "3.0.0", "", null, null); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(firstExtension)); assertEquals("Required field [minimum opensearch version] is missing in the request", exception.getMessage()); @@ -376,7 +376,7 @@ public void testExtensionDependency() throws Exception { } public void testInitialize() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); @@ -418,7 +418,7 @@ public void testInitialize() throws Exception { public void testHandleRegisterRestActionsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -432,7 +432,7 @@ public void testHandleRegisterRestActionsRequest() throws Exception { } public void testHandleRegisterSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -448,7 +448,7 @@ public void testHandleRegisterSettingsRequest() throws Exception { } public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -463,7 +463,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Excep } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -478,7 +478,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() th } public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET", "PUT /bar", "POST /baz"); @@ -492,7 +492,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exceptio } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET /foo", "PUT /bar", "POST /baz"); @@ -506,7 +506,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throw } public void testHandleExtensionRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); ExtensionRequest clusterStateRequest = new ExtensionRequest(ExtensionRequestProto.RequestType.REQUEST_EXTENSION_CLUSTER_STATE); @@ -660,7 +660,7 @@ public void testEnvironmentSettingsDefaultValue() throws Exception { } public void testAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); List> componentSettings = List.of( @@ -704,7 +704,7 @@ public void testAddSettingsUpdateConsumerRequest() throws Exception { } public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); List> componentSettings = List.of( @@ -724,7 +724,7 @@ public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { } public void testUpdateSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); Setting componentSetting = Setting.boolSetting("falseSetting", false, Property.Dynamic); @@ -753,7 +753,7 @@ public void testUpdateSettingsRequest() throws Exception { public void testRegisterHandler() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); TransportService mockTransportService = spy( new TransportService( @@ -781,7 +781,7 @@ public void testRegisterHandler() throws Exception { } public void testIncompatibleExtensionRegistration() throws IOException { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); Extension firstExtension = new Extension( "firstExtension", "uniqueid1", @@ -822,7 +822,7 @@ public List> getExtensionSettings() { extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( @@ -860,7 +860,7 @@ public void testAdditionalExtensionSettingsForExtensionWithoutCustomSettingSet() extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 87767978147cd..cdddf8e9be1be 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -21,6 +21,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.extensions.ExtensionsSettings; +import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestRequest; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; @@ -118,7 +119,7 @@ public void testRestInitializeExtensionActionResponse() throws Exception { } public void testRestInitializeExtensionActionFailure() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(extensionsManager); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"\",\"hostAddress\":\"127.0.0.1\"," @@ -151,7 +152,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettings() th Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager extensionsManager = new ExtensionsManager( + Set.of(boolSetting, stringSetting, intSetting, listSetting), + new IdentityService(Settings.EMPTY, List.of()) + ); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: @@ -198,7 +202,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsing Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager extensionsManager = new ExtensionsManager( + Set.of(boolSetting, stringSetting, intSetting, listSetting), + new IdentityService(Settings.EMPTY, List.of()) + ); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index 992e37b3f120a..ee36ea170e270 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -122,7 +122,7 @@ public void setup() throws Exception { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of()) + new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) ); identityService = new IdentityService(Settings.EMPTY, new ArrayList<>()); dynamicActionRegistry = actionModule.getDynamicActionRegistry(); From 38db1dfb829b8ea2089305ae111374aa225104f1 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 22 Sep 2023 12:42:54 -0700 Subject: [PATCH 061/388] Reenable multipart repository within multipart ITs (#10178) Signed-off-by: Kunal Kotwani --- .../RemoteStoreBaseIntegTestCase.java | 91 ++++++++++--------- .../multipart/RemoteStoreMultipartIT.java | 18 ++++ 2 files changed, 64 insertions(+), 45 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 621fb262c0a91..157f8e41fee24 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -27,8 +27,6 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.node.Node; -import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; @@ -168,6 +166,29 @@ public static Settings remoteStoreClusterSettings(String name, Path path) { return remoteStoreClusterSettings(name, path, name, path); } + public static Settings remoteStoreClusterSettings( + String segmentRepoName, + Path segmentRepoPath, + String segmentRepoType, + String translogRepoName, + Path translogRepoPath, + String translogRepoType + ) { + Settings.Builder settingsBuilder = Settings.builder(); + settingsBuilder.put( + buildRemoteStoreNodeAttributes( + segmentRepoName, + segmentRepoPath, + segmentRepoType, + translogRepoName, + translogRepoPath, + translogRepoType, + false + ) + ); + return settingsBuilder.build(); + } + public static Settings remoteStoreClusterSettings( String segmentRepoName, Path segmentRepoPath, @@ -185,6 +206,26 @@ public static Settings buildRemoteStoreNodeAttributes( String translogRepoName, Path translogRepoPath, boolean withRateLimiterAttributes + ) { + return buildRemoteStoreNodeAttributes( + segmentRepoName, + segmentRepoPath, + FsRepository.TYPE, + translogRepoName, + translogRepoPath, + FsRepository.TYPE, + withRateLimiterAttributes + ); + } + + public static Settings buildRemoteStoreNodeAttributes( + String segmentRepoName, + Path segmentRepoPath, + String segmentRepoType, + String translogRepoName, + Path translogRepoPath, + String translogRepoType, + boolean withRateLimiterAttributes ) { String segmentRepoTypeAttributeKey = String.format( Locale.getDefault(), @@ -219,13 +260,13 @@ public static Settings buildRemoteStoreNodeAttributes( Settings.Builder settings = Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) + .put(segmentRepoTypeAttributeKey, segmentRepoType) .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) - .put(translogRepoTypeAttributeKey, FsRepository.TYPE) + .put(translogRepoTypeAttributeKey, translogRepoType) .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoTypeAttributeKey, segmentRepoType) .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); if (withRateLimiterAttributes) { @@ -310,46 +351,6 @@ public void assertRemoteStoreRepositoryOnAllNodes(String repositoryName) { } } - public Settings buildClusterSettingsWith() { - String segmentRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - REPOSITORY_NAME - ); - String segmentRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - REPOSITORY_NAME - ); - String translogRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - REPOSITORY_2_NAME - ); - String translogRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - REPOSITORY_2_NAME - ); - return Settings.builder() - .put( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, - REPOSITORY_NAME - ) - .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) - .put(segmentRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) - .put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) - .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "2kb") - .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES) - .put( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, - REPOSITORY_2_NAME - ) - .put(translogRepoTypeAttributeKey, FsRepository.TYPE) - .put(translogRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) - .build(); - } - public static int getFileCount(Path path) throws Exception { final AtomicInteger filesExisting = new AtomicInteger(0); Files.walkFileTree(path, new SimpleFileVisitor<>() { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java index 21f48ba99e651..3ceb09e44ae36 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -46,8 +46,26 @@ protected Collection> nodePlugins() { return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); } + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put( + remoteStoreClusterSettings( + REPOSITORY_NAME, + segmentRepoPath, + MockFsRepositoryPlugin.TYPE, + REPOSITORY_2_NAME, + translogRepoPath, + MockFsRepositoryPlugin.TYPE + ) + ) + .build(); + } + @Before public void setup() { + clusterSettingsSuppliedByTest = true; overrideBuildRepositoryMetadata = false; repositoryLocation = randomRepoPath(); compress = randomBoolean(); From c4c4ad84d995f75f8749a0f99aa8a1ecc3b71760 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 22 Sep 2023 16:14:12 -0700 Subject: [PATCH 062/388] Add support for encrypted async blob read (#10131) * Add support for encrypted async blob read Signed-off-by: Kunal Kotwani * Add async blob read support for encrypted containers Signed-off-by: Kunal Kotwani --------- Signed-off-by: Kunal Kotwani --- CHANGELOG.md | 1 + ...syncMultiStreamEncryptedBlobContainer.java | 77 +++++++++-- .../blobstore/EncryptedBlobContainer.java | 2 +- .../blobstore/stream/read/ReadContext.java | 6 + ...ultiStreamEncryptedBlobContainerTests.java | 121 ++++++++++++++++++ .../read/listener/ListenerTestUtils.java | 2 +- 6 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 85211cff7f1bd..ca45edad0f11f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681)) - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694)) - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666)) +- Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java index 07a0b49df47ff..c64dc6b9e3ae4 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java @@ -12,12 +12,14 @@ import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.DecryptedRangedStreamProvider; import org.opensearch.common.io.InputStreamContainer; import org.opensearch.core.action.ActionListener; -import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import java.nio.file.Path; +import java.io.InputStream; +import java.util.List; +import java.util.stream.Collectors; /** * EncryptedBlobContainer is an encrypted BlobContainer that is backed by a @@ -44,12 +46,17 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener comp @Override public void readBlobAsync(String blobName, ActionListener listener) { - throw new UnsupportedOperationException(); - } - - @Override - public void asyncBlobDownload(String blobName, Path fileLocation, ThreadPool threadPool, ActionListener completionListener) { - throw new UnsupportedOperationException(); + try { + final U cryptoContext = cryptoHandler.loadEncryptionMetadata(getEncryptedHeaderContentSupplier(blobName)); + ActionListener decryptingCompletionListener = ActionListener.map( + listener, + readContext -> new DecryptedReadContext<>(readContext, cryptoHandler, cryptoContext) + ); + + blobContainer.readBlobAsync(blobName, decryptingCompletionListener); + } catch (Exception e) { + listener.onFailure(e); + } } @Override @@ -108,4 +115,58 @@ public InputStreamContainer provideStream(int partNumber) throws IOException { } } + + /** + * DecryptedReadContext decrypts the encrypted {@link ReadContext} by acting as a transformation wrapper around + * the encrypted object + * @param Encryption Metadata / CryptoContext for the {@link CryptoHandler} instance + * @param Parsed Encryption Metadata / CryptoContext for the {@link CryptoHandler} instance + */ + static class DecryptedReadContext extends ReadContext { + + private final CryptoHandler cryptoHandler; + private final U cryptoContext; + private Long blobSize; + + public DecryptedReadContext(ReadContext readContext, CryptoHandler cryptoHandler, U cryptoContext) { + super(readContext); + this.cryptoHandler = cryptoHandler; + this.cryptoContext = cryptoContext; + } + + @Override + public long getBlobSize() { + // initializes the value lazily + if (blobSize == null) { + this.blobSize = this.cryptoHandler.estimateDecryptedLength(cryptoContext, super.getBlobSize()); + } + return this.blobSize; + } + + @Override + public List getPartStreams() { + return super.getPartStreams().stream().map(this::decryptInputStreamContainer).collect(Collectors.toList()); + } + + /** + * Transforms an encrypted {@link InputStreamContainer} to a decrypted instance + * @param inputStreamContainer encrypted input stream container instance + * @return decrypted input stream container instance + */ + private InputStreamContainer decryptInputStreamContainer(InputStreamContainer inputStreamContainer) { + long startOfStream = inputStreamContainer.getOffset(); + long endOfStream = startOfStream + inputStreamContainer.getContentLength() - 1; + DecryptedRangedStreamProvider decryptedStreamProvider = cryptoHandler.createDecryptingStreamOfRange( + cryptoContext, + startOfStream, + endOfStream + ); + + long adjustedPos = decryptedStreamProvider.getAdjustedRange()[0]; + long adjustedLength = decryptedStreamProvider.getAdjustedRange()[1] - adjustedPos + 1; + final InputStream decryptedStream = decryptedStreamProvider.getDecryptedStreamProvider() + .apply(inputStreamContainer.getInputStream()); + return new InputStreamContainer(decryptedStream, adjustedLength, adjustedPos); + } + } } diff --git a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobContainer.java index 475d891ea9336..d0933741339d9 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobContainer.java @@ -50,7 +50,7 @@ public InputStream readBlob(String blobName) throws IOException { return cryptoHandler.createDecryptingStream(inputStream); } - private EncryptedHeaderContentSupplier getEncryptedHeaderContentSupplier(String blobName) { + EncryptedHeaderContentSupplier getEncryptedHeaderContentSupplier(String blobName) { return (start, end) -> { byte[] buffer; int length = (int) (end - start + 1); diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java index 4ba17959f8040..2c305fb03c475 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java +++ b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java @@ -28,6 +28,12 @@ public ReadContext(long blobSize, List partStreams, String this.blobChecksum = blobChecksum; } + public ReadContext(ReadContext readContext) { + this.blobSize = readContext.blobSize; + this.partStreams = readContext.partStreams; + this.blobChecksum = readContext.blobChecksum; + } + public String getBlobChecksum() { return blobChecksum; } diff --git a/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java new file mode 100644 index 0000000000000..947a4f9b1c9ab --- /dev/null +++ b/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.blobstore; + +import org.opensearch.common.Randomness; +import org.opensearch.common.blobstore.stream.read.ReadContext; +import org.opensearch.common.blobstore.stream.read.listener.ListenerTestUtils; +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.DecryptedRangedStreamProvider; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.core.action.ActionListener; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.List; +import java.util.function.UnaryOperator; + +import org.mockito.Mockito; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AsyncMultiStreamEncryptedBlobContainerTests extends OpenSearchTestCase { + + // Tests the happy path scenario for decrypting a read context + @SuppressWarnings("unchecked") + public void testReadBlobAsync() throws Exception { + String testBlobName = "testBlobName"; + int size = 100; + + // Mock objects needed for the test + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); + CryptoHandler cryptoHandler = mock(CryptoHandler.class); + Object cryptoContext = mock(Object.class); + when(cryptoHandler.loadEncryptionMetadata(any())).thenReturn(cryptoContext); + when(cryptoHandler.estimateDecryptedLength(any(), anyLong())).thenReturn((long) size); + long[] adjustedRanges = { 0, size - 1 }; + DecryptedRangedStreamProvider rangedStreamProvider = new DecryptedRangedStreamProvider(adjustedRanges, UnaryOperator.identity()); + when(cryptoHandler.createDecryptingStreamOfRange(eq(cryptoContext), anyLong(), anyLong())).thenReturn(rangedStreamProvider); + + // Objects needed for API call + final byte[] data = new byte[size]; + Randomness.get().nextBytes(data); + final InputStreamContainer inputStreamContainer = new InputStreamContainer(new ByteArrayInputStream(data), data.length, 0); + final ListenerTestUtils.CountingCompletionListener completionListener = + new ListenerTestUtils.CountingCompletionListener<>(); + final ReadContext readContext = new ReadContext(size, List.of(inputStreamContainer), null); + + Mockito.doAnswer(invocation -> { + ActionListener readContextActionListener = invocation.getArgument(1); + readContextActionListener.onResponse(readContext); + return null; + }).when(blobContainer).readBlobAsync(eq(testBlobName), any()); + + AsyncMultiStreamEncryptedBlobContainer asyncMultiStreamEncryptedBlobContainer = + new AsyncMultiStreamEncryptedBlobContainer<>(blobContainer, cryptoHandler); + asyncMultiStreamEncryptedBlobContainer.readBlobAsync(testBlobName, completionListener); + + // Assert results + ReadContext response = completionListener.getResponse(); + assertEquals(0, completionListener.getFailureCount()); + assertEquals(1, completionListener.getResponseCount()); + assertNull(completionListener.getException()); + + assertTrue(response instanceof AsyncMultiStreamEncryptedBlobContainer.DecryptedReadContext); + assertEquals(1, response.getNumberOfParts()); + assertEquals(size, response.getBlobSize()); + + InputStreamContainer responseContainer = response.getPartStreams().get(0); + assertEquals(0, responseContainer.getOffset()); + assertEquals(size, responseContainer.getContentLength()); + assertEquals(100, responseContainer.getInputStream().available()); + } + + // Tests the exception scenario for decrypting a read context + @SuppressWarnings("unchecked") + public void testReadBlobAsyncException() throws Exception { + String testBlobName = "testBlobName"; + int size = 100; + + // Mock objects needed for the test + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); + CryptoHandler cryptoHandler = mock(CryptoHandler.class); + when(cryptoHandler.loadEncryptionMetadata(any())).thenThrow(new IOException()); + + // Objects needed for API call + final byte[] data = new byte[size]; + Randomness.get().nextBytes(data); + final InputStreamContainer inputStreamContainer = new InputStreamContainer(new ByteArrayInputStream(data), data.length, 0); + final ListenerTestUtils.CountingCompletionListener completionListener = + new ListenerTestUtils.CountingCompletionListener<>(); + final ReadContext readContext = new ReadContext(size, List.of(inputStreamContainer), null); + + Mockito.doAnswer(invocation -> { + ActionListener readContextActionListener = invocation.getArgument(1); + readContextActionListener.onResponse(readContext); + return null; + }).when(blobContainer).readBlobAsync(eq(testBlobName), any()); + + AsyncMultiStreamEncryptedBlobContainer asyncMultiStreamEncryptedBlobContainer = + new AsyncMultiStreamEncryptedBlobContainer<>(blobContainer, cryptoHandler); + asyncMultiStreamEncryptedBlobContainer.readBlobAsync(testBlobName, completionListener); + + // Assert results + assertEquals(1, completionListener.getFailureCount()); + assertEquals(0, completionListener.getResponseCount()); + assertNull(completionListener.getResponse()); + assertTrue(completionListener.getException() instanceof IOException); + } + +} diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ListenerTestUtils.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ListenerTestUtils.java index 1e9450c83e3ab..a3a32f6db2148 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ListenerTestUtils.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ListenerTestUtils.java @@ -19,7 +19,7 @@ public class ListenerTestUtils { * CountingCompletionListener acts as a verification instance for wrapping listener based calls. * Keeps track of the last response, failure and count of response and failure invocations. */ - static class CountingCompletionListener implements ActionListener { + public static class CountingCompletionListener implements ActionListener { private int responseCount; private int failureCount; private T response; From 9e906714ebfd2c1a89d6249362fe1cc1cff4c548 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 22 Sep 2023 16:56:17 -0700 Subject: [PATCH 063/388] Add async segment file download support from remote store (#9710) Signed-off-by: Kunal Kotwani --- .../multipart/RemoteStoreMultipartIT.java | 1 + .../org/opensearch/index/IndexService.java | 5 +- .../opensearch/index/shard/IndexShard.java | 60 ++++++++-- .../store/RemoteSegmentStoreDirectory.java | 37 ++++++ .../org/opensearch/index/store/Store.java | 20 +++- .../RemoteStoreReplicationSource.java | 32 ++++- .../index/shard/IndexShardTests.java | 2 +- .../RemoteSegmentStoreDirectoryTests.java | 112 ++++++++++++++++++ .../opensearch/index/store/StoreTests.java | 23 +++- .../index/shard/IndexShardTestCase.java | 13 +- 10 files changed, 274 insertions(+), 31 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java index 3ceb09e44ae36..98fab139f4902 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -104,6 +104,7 @@ public RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String nam } else { return super.buildRepositoryMetadata(node, name); } + } public void testRateLimitedRemoteUploads() throws Exception { diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 38456ad3e43a5..ca0cc307e460b 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -481,7 +481,7 @@ public synchronized IndexShard createShard( Store remoteStore = null; if (this.indexSettings.isRemoteStoreEnabled()) { Directory remoteDirectory = remoteDirectoryFactory.newDirectory(this.indexSettings, path); - remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, lock, Store.OnClose.EMPTY); + remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, lock, Store.OnClose.EMPTY, path); } Directory directory = directoryFactory.newDirectory(this.indexSettings, path); @@ -490,7 +490,8 @@ public synchronized IndexShard createShard( this.indexSettings, directory, lock, - new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId)) + new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId)), + path ); eventListener.onStoreCreated(shardId); indexShard = new IndexShard( diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 34c5ed2112482..6ab2110370f52 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -62,6 +62,8 @@ import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.opensearch.action.admin.indices.upgrade.post.UpgradeRequest; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.replication.PendingReplicationActions; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.cluster.metadata.DataStream; @@ -96,7 +98,6 @@ import org.opensearch.common.util.concurrent.RunOnce; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.common.util.set.Sets; import org.opensearch.core.Assertions; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeValue; @@ -199,6 +200,7 @@ import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -854,7 +856,7 @@ public void relocated( synchronized (mutex) { verifyRelocatingState(); replicationTracker.completeRelocationHandoff(); // make changes to primaryMode and relocated flag only under - // mutex + // mutex } } catch (final Exception e) { try { @@ -3790,7 +3792,7 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro replicationTracker::isPrimaryMode, translogFactorySupplier.apply(indexSettings, shardRouting), isTimeSeriesDescSortOptimizationEnabled() ? DataStream.TIMESERIES_LEAF_SORTER : null // DESC @timestamp default order for - // timeseries + // timeseries ); } @@ -4853,39 +4855,71 @@ private String copySegmentFiles( Map uploadedSegments, boolean overrideLocal ) throws IOException { - List downloadedSegments = new ArrayList<>(); - List skippedSegments = new ArrayList<>(); + Set toDownloadSegments = new HashSet<>(); + Set skippedSegments = new HashSet<>(); String segmentNFile = null; + try { - Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); if (overrideLocal) { - for (String file : localSegmentFiles) { + for (String file : storeDirectory.listAll()) { storeDirectory.deleteFile(file); } } + for (String file : uploadedSegments.keySet()) { long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); - downloadedSegments.add(file); + toDownloadSegments.add(file); } else { skippedSegments.add(file); } - if (targetRemoteDirectory != null) { - targetRemoteDirectory.copyFrom(storeDirectory, file, file, IOContext.DEFAULT); - } + if (file.startsWith(IndexFileNames.SEGMENTS)) { assert segmentNFile == null : "There should be only one SegmentInfosSnapshot file"; segmentNFile = file; } } + + if (toDownloadSegments.isEmpty() == false) { + try { + final PlainActionFuture completionListener = PlainActionFuture.newFuture(); + downloadSegments(storeDirectory, sourceRemoteDirectory, targetRemoteDirectory, toDownloadSegments, completionListener); + completionListener.actionGet(); + } catch (Exception e) { + throw new IOException("Error occurred when downloading segments from remote store", e); + } + } } finally { - logger.trace("Downloaded segments here: {}", downloadedSegments); + logger.trace("Downloaded segments here: {}", toDownloadSegments); logger.trace("Skipped download for segments here: {}", skippedSegments); } + return segmentNFile; } + private void downloadSegments( + Directory storeDirectory, + RemoteSegmentStoreDirectory sourceRemoteDirectory, + RemoteSegmentStoreDirectory targetRemoteDirectory, + Set toDownloadSegments, + ActionListener completionListener + ) { + final Path indexPath = store.shardPath() == null ? null : store.shardPath().resolveIndex(); + final GroupedActionListener batchDownloadListener = new GroupedActionListener<>( + ActionListener.map(completionListener, v -> null), + toDownloadSegments.size() + ); + + final ActionListener segmentsDownloadListener = ActionListener.map(batchDownloadListener, fileName -> { + if (targetRemoteDirectory != null) { + targetRemoteDirectory.copyFrom(storeDirectory, fileName, fileName, IOContext.DEFAULT); + } + return null; + }); + + toDownloadSegments.forEach(file -> sourceRemoteDirectory.copyTo(file, storeDirectory, indexPath, segmentsDownloadListener)); + } + private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) { try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) { if (checksum == CodecUtil.retrieveChecksum(indexInput)) { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 266552cea93d9..a68fed5e302bd 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -24,6 +24,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Version; import org.opensearch.common.UUIDs; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.core.action.ActionListener; @@ -40,6 +41,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -434,6 +436,41 @@ public void copyFrom(Directory from, String src, IOContext context, ActionListen } } + /** + * Copies an existing {@code source} file from this directory to a non-existent file (also + * named {@code source}) in either {@code destinationDirectory} or {@code destinationPath}. + * If the blob container backing this directory supports multipart downloads, the {@code source} + * file will be downloaded (potentially in multiple concurrent parts) directly to + * {@code destinationPath}. This method will return immediately and {@code fileCompletionListener} + * will be notified upon completion. + *

+ * If multipart downloads are not supported, then {@code source} file will be copied to a file named + * {@code source} in a single part to {@code destinationDirectory}. The download will happen on the + * calling thread and {@code fileCompletionListener} will be notified synchronously before this + * method returns. + * + * @param source The source file name + * @param destinationDirectory The destination directory (if multipart is not supported) + * @param destinationPath The destination path (if multipart is supported) + * @param fileCompletionListener The listener to notify of completion + */ + public void copyTo(String source, Directory destinationDirectory, Path destinationPath, ActionListener fileCompletionListener) { + final String blobName = getExistingRemoteFilename(source); + if (destinationPath != null && remoteDataDirectory.getBlobContainer() instanceof AsyncMultiStreamBlobContainer) { + final AsyncMultiStreamBlobContainer blobContainer = (AsyncMultiStreamBlobContainer) remoteDataDirectory.getBlobContainer(); + final Path destinationFilePath = destinationPath.resolve(source); + blobContainer.asyncBlobDownload(blobName, destinationFilePath, threadPool, fileCompletionListener); + } else { + // Fallback to older mechanism of downloading the file + try { + destinationDirectory.copyFrom(this, source, source, IOContext.DEFAULT); + fileCompletionListener.onResponse(source); + } catch (IOException e) { + fileCompletionListener.onFailure(e); + } + } + } + /** * This acquires a lock on a given commit by creating a lock file in lock directory using {@code FileLockInfo} * diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index b3ea2cdd02e21..285241ba89996 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -66,6 +66,7 @@ import org.apache.lucene.util.Version; import org.opensearch.ExceptionsHelper; import org.opensearch.common.UUIDs; +import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.logging.Loggers; import org.opensearch.common.lucene.Lucene; @@ -92,6 +93,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.AbstractIndexShardComponent; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.ShardPath; import org.opensearch.index.translog.Translog; import java.io.Closeable; @@ -179,6 +181,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref private final ReentrantReadWriteLock metadataLock = new ReentrantReadWriteLock(); private final ShardLock shardLock; private final OnClose onClose; + private final ShardPath shardPath; // used to ref count files when a new Reader is opened for PIT/Scroll queries // prevents segment files deletion until the PIT/Scroll expires or is discarded @@ -192,10 +195,17 @@ protected void closeInternal() { }; public Store(ShardId shardId, IndexSettings indexSettings, Directory directory, ShardLock shardLock) { - this(shardId, indexSettings, directory, shardLock, OnClose.EMPTY); + this(shardId, indexSettings, directory, shardLock, OnClose.EMPTY, null); } - public Store(ShardId shardId, IndexSettings indexSettings, Directory directory, ShardLock shardLock, OnClose onClose) { + public Store( + ShardId shardId, + IndexSettings indexSettings, + Directory directory, + ShardLock shardLock, + OnClose onClose, + ShardPath shardPath + ) { super(shardId, indexSettings); final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING); logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval); @@ -203,6 +213,7 @@ public Store(ShardId shardId, IndexSettings indexSettings, Directory directory, this.directory = new StoreDirectory(sizeCachingDir, Loggers.getLogger("index.store.deletes", shardId)); this.shardLock = shardLock; this.onClose = onClose; + this.shardPath = shardPath; assert onClose != null; assert shardLock != null; assert shardLock.getShardId().equals(shardId); @@ -213,6 +224,11 @@ public Directory directory() { return directory; } + @InternalApi + public ShardPath shardPath() { + return shardPath; + } + /** * Returns the last committed segments info for this store * diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java index 7252fea044a02..64aacd68f2490 100644 --- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java @@ -13,18 +13,20 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; -import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Version; +import org.opensearch.action.support.GroupedActionListener; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.core.action.ActionListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; +import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -106,31 +108,49 @@ public void getSegmentFiles( logger.trace("Downloading segments files from remote store {}", filesToFetch); RemoteSegmentMetadata remoteSegmentMetadata = remoteDirectory.readLatestMetadataFile(); - List downloadedSegments = new ArrayList<>(); + List toDownloadSegments = new ArrayList<>(); Collection directoryFiles = List.of(indexShard.store().directory().listAll()); if (remoteSegmentMetadata != null) { try { indexShard.store().incRef(); indexShard.remoteStore().incRef(); final Directory storeDirectory = indexShard.store().directory(); + final ShardPath shardPath = indexShard.shardPath(); for (StoreFileMetadata fileMetadata : filesToFetch) { String file = fileMetadata.name(); assert directoryFiles.contains(file) == false : "Local store already contains the file " + file; - storeDirectory.copyFrom(remoteDirectory, file, file, IOContext.DEFAULT); - downloadedSegments.add(fileMetadata); + toDownloadSegments.add(fileMetadata); } - logger.trace("Downloaded segments from remote store {}", downloadedSegments); + downloadSegments(storeDirectory, remoteDirectory, toDownloadSegments, shardPath, listener); + logger.trace("Downloaded segments from remote store {}", toDownloadSegments); } finally { indexShard.store().decRef(); indexShard.remoteStore().decRef(); } } - listener.onResponse(new GetSegmentFilesResponse(downloadedSegments)); } catch (Exception e) { listener.onFailure(e); } } + private void downloadSegments( + Directory storeDirectory, + RemoteSegmentStoreDirectory remoteStoreDirectory, + List toDownloadSegments, + ShardPath shardPath, + ActionListener completionListener + ) { + final Path indexPath = shardPath == null ? null : shardPath.resolveIndex(); + final GroupedActionListener batchDownloadListener = new GroupedActionListener<>( + ActionListener.map(completionListener, v -> new GetSegmentFilesResponse(toDownloadSegments)), + toDownloadSegments.size() + ); + ActionListener segmentsDownloadListener = ActionListener.map(batchDownloadListener, result -> null); + toDownloadSegments.forEach( + fileMetadata -> remoteStoreDirectory.copyTo(fileMetadata.name(), storeDirectory, indexPath, segmentsDownloadListener) + ); + } + @Override public String getDescription() { return "RemoteStoreReplicationSource"; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index cd272d8f626d0..b2eb41828a4df 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -1780,7 +1780,7 @@ public Set getPendingDeletions() throws IOException { } }; - try (Store store = createStore(shardId, new IndexSettings(metadata, Settings.EMPTY), directory)) { + try (Store store = createStore(shardId, new IndexSettings(metadata, Settings.EMPTY), directory, shardPath)) { IndexShard shard = newShard( shardRouting, shardPath, diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 2948528ad82e1..b016e8be67799 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -28,6 +28,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; @@ -46,6 +47,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -61,6 +63,8 @@ import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes; import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata; import static org.hamcrest.CoreMatchers.is; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -516,6 +520,114 @@ public void onFailure(Exception e) {} storeDirectory.close(); } + public void testCopyFilesToMultipart() throws Exception { + Settings settings = Settings.builder().build(); + FeatureFlags.initializeFeatureFlags(settings); + + String filename = "_0.cfe"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = mock(Directory.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); + when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); + + Mockito.doAnswer(invocation -> { + ActionListener completionListener = invocation.getArgument(3); + completionListener.onResponse(invocation.getArgument(0)); + return null; + }).when(blobContainer).asyncBlobDownload(any(), any(), any(), any()); + + CountDownLatch downloadLatch = new CountDownLatch(1); + ActionListener completionListener = new ActionListener() { + @Override + public void onResponse(String unused) { + downloadLatch.countDown(); + } + + @Override + public void onFailure(Exception e) {} + }; + Path path = createTempDir(); + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, completionListener); + assertTrue(downloadLatch.await(5000, TimeUnit.SECONDS)); + verify(blobContainer, times(1)).asyncBlobDownload(contains(filename), eq(path.resolve(filename)), any(), any()); + verify(storeDirectory, times(0)).copyFrom(any(), any(), any(), any()); + } + + public void testCopyFilesTo() throws Exception { + String filename = "_0.cfe"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = mock(Directory.class); + CountDownLatch downloadLatch = new CountDownLatch(1); + ActionListener completionListener = new ActionListener<>() { + @Override + public void onResponse(String unused) { + downloadLatch.countDown(); + } + + @Override + public void onFailure(Exception e) {} + }; + Path path = createTempDir(); + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, completionListener); + assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); + verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); + } + + public void testCopyFilesToEmptyPath() throws Exception { + String filename = "_0.cfe"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = mock(Directory.class); + AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); + when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); + + CountDownLatch downloadLatch = new CountDownLatch(1); + ActionListener completionListener = new ActionListener<>() { + @Override + public void onResponse(String unused) { + downloadLatch.countDown(); + } + + @Override + public void onFailure(Exception e) {} + }; + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, null, completionListener); + assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); + verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); + } + + public void testCopyFilesToException() throws Exception { + String filename = "_0.cfe"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = mock(Directory.class); + Mockito.doThrow(new IOException()) + .when(storeDirectory) + .copyFrom(any(Directory.class), anyString(), anyString(), any(IOContext.class)); + CountDownLatch downloadLatch = new CountDownLatch(1); + ActionListener completionListener = new ActionListener<>() { + @Override + public void onResponse(String unused) { + + } + + @Override + public void onFailure(Exception e) { + downloadLatch.countDown(); + } + }; + Path path = createTempDir(); + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, completionListener); + assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); + verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); + } + public void testCopyFilesFromMultipartIOException() throws Exception { String filename = "_100.si"; AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); diff --git a/server/src/test/java/org/opensearch/index/store/StoreTests.java b/server/src/test/java/org/opensearch/index/store/StoreTests.java index 8395b3e8ac08e..d7d326b325cc6 100644 --- a/server/src/test/java/org/opensearch/index/store/StoreTests.java +++ b/server/src/test/java/org/opensearch/index/store/StoreTests.java @@ -84,6 +84,7 @@ import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.shard.ShardPath; import org.opensearch.index.translog.Translog; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata; @@ -798,7 +799,7 @@ public void testOnCloseCallback() throws IOException { assertEquals(shardId, theLock.getShardId()); assertEquals(lock, theLock); count.incrementAndGet(); - }); + }, null); assertEquals(count.get(), 0); final int iters = randomIntBetween(1, 10); @@ -809,6 +810,26 @@ public void testOnCloseCallback() throws IOException { assertEquals(count.get(), 1); } + public void testStoreShardPath() { + final ShardId shardId = new ShardId("index", "_na_", 1); + final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT) + .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMinutes(0)) + .build(); + final Path path = createTempDir().resolve(shardId.getIndex().getUUID()).resolve(String.valueOf(shardId.id())); + final ShardPath shardPath = new ShardPath(false, path, path, shardId); + final Store store = new Store( + shardId, + IndexSettingsModule.newIndexSettings("index", settings), + StoreTests.newDirectory(random()), + new DummyShardLock(shardId), + Store.OnClose.EMPTY, + shardPath + ); + assertEquals(shardPath, store.shardPath()); + store.close(); + } + public void testStoreStats() throws IOException { final ShardId shardId = new ShardId("index", "_na_", 1); Settings settings = Settings.builder() diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 43f2cce668e81..7a54c32248218 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -272,11 +272,11 @@ public Settings threadPoolSettings() { } protected Store createStore(IndexSettings indexSettings, ShardPath shardPath) throws IOException { - return createStore(shardPath.getShardId(), indexSettings, newFSDirectory(shardPath.resolveIndex())); + return createStore(shardPath.getShardId(), indexSettings, newFSDirectory(shardPath.resolveIndex()), shardPath); } - protected Store createStore(ShardId shardId, IndexSettings indexSettings, Directory directory) throws IOException { - return new Store(shardId, indexSettings, directory, new DummyShardLock(shardId)); + protected Store createStore(ShardId shardId, IndexSettings indexSettings, Directory directory, ShardPath shardPath) throws IOException { + return new Store(shardId, indexSettings, directory, new DummyShardLock(shardId), Store.OnClose.EMPTY, shardPath); } protected Releasable acquirePrimaryOperationPermitBlockingly(IndexShard indexShard) throws ExecutionException, InterruptedException { @@ -654,7 +654,7 @@ protected IndexShard newShard( remotePath = createTempDir(); } - remoteStore = createRemoteStore(remotePath, routing, indexMetadata); + remoteStore = createRemoteStore(remotePath, routing, indexMetadata, shardPath); remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, indexSettings.getSettings()); BlobStoreRepository repo = createRepository(remotePath); @@ -768,11 +768,12 @@ protected RepositoriesService createRepositoriesService() { return repositoriesService; } - protected Store createRemoteStore(Path path, ShardRouting shardRouting, IndexMetadata metadata) throws IOException { + protected Store createRemoteStore(Path path, ShardRouting shardRouting, IndexMetadata metadata, ShardPath shardPath) + throws IOException { Settings nodeSettings = Settings.builder().put("node.name", shardRouting.currentNodeId()).build(); ShardId shardId = shardRouting.shardId(); RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = createRemoteSegmentStoreDirectory(shardId, path); - return createStore(shardId, new IndexSettings(metadata, nodeSettings), remoteSegmentStoreDirectory); + return createStore(shardId, new IndexSettings(metadata, nodeSettings), remoteSegmentStoreDirectory, shardPath); } protected RemoteSegmentStoreDirectory createRemoteSegmentStoreDirectory(ShardId shardId, Path path) throws IOException { From 1dde018a1aa9cb606c274a4119348de285b80a65 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Sat, 23 Sep 2023 09:16:35 -0700 Subject: [PATCH 064/388] Honor max segment size during only_expunge_deletes (#10036) --- CHANGELOG.md | 1 + .../index/OpenSearchTieredMergePolicy.java | 6 +-- .../OpenSearchTieredMergePolicyTests.java | 37 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca45edad0f11f..d4d664e9154bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) - Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916)) - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) +- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036)) - Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/index/OpenSearchTieredMergePolicy.java b/server/src/main/java/org/opensearch/index/OpenSearchTieredMergePolicy.java index 637282374de73..df1666e72f2ee 100644 --- a/server/src/main/java/org/opensearch/index/OpenSearchTieredMergePolicy.java +++ b/server/src/main/java/org/opensearch/index/OpenSearchTieredMergePolicy.java @@ -42,7 +42,7 @@ /** * Wrapper around {@link TieredMergePolicy} which doesn't respect - * {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges. + * {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges, but DOES respect it on only_expunge_deletes. * See https://issues.apache.org/jira/browse/LUCENE-7976. * * @opensearch.internal @@ -71,7 +71,7 @@ public MergeSpecification findForcedMerges( @Override public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException { - return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext); + return regularMergePolicy.findForcedDeletesMerges(infos, mergeContext); } public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) { @@ -80,7 +80,7 @@ public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) { } public double getForceMergeDeletesPctAllowed() { - return forcedMergePolicy.getForceMergeDeletesPctAllowed(); + return regularMergePolicy.getForceMergeDeletesPctAllowed(); } public void setFloorSegmentMB(double mbFrac) { diff --git a/server/src/test/java/org/opensearch/index/OpenSearchTieredMergePolicyTests.java b/server/src/test/java/org/opensearch/index/OpenSearchTieredMergePolicyTests.java index cbc439041666f..614dacd457782 100644 --- a/server/src/test/java/org/opensearch/index/OpenSearchTieredMergePolicyTests.java +++ b/server/src/test/java/org/opensearch/index/OpenSearchTieredMergePolicyTests.java @@ -32,9 +32,18 @@ package org.opensearch.index; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; +import java.util.Collections; +import java.util.Set; + public class OpenSearchTieredMergePolicyTests extends OpenSearchTestCase { public void testDefaults() { @@ -80,4 +89,32 @@ public void testSetDeletesPctAllowed() { policy.setDeletesPctAllowed(42); assertEquals(42, policy.regularMergePolicy.getDeletesPctAllowed(), 0); } + + public void testFindDeleteMergesReturnsNullOnEmptySegmentInfos() throws IOException { + MergePolicy.MergeSpecification mergeSpecification = new OpenSearchTieredMergePolicy().findForcedDeletesMerges( + new SegmentInfos(Version.LATEST.major), + new MergePolicy.MergeContext() { + @Override + public int numDeletesToMerge(SegmentCommitInfo info) { + return 0; + } + + @Override + public int numDeletedDocs(SegmentCommitInfo info) { + return 0; + } + + @Override + public InfoStream getInfoStream() { + return InfoStream.NO_OUTPUT; + } + + @Override + public Set getMergingSegments() { + return Collections.emptySet(); + } + } + ); + assertNull(mergeSpecification); + } } From b83f4333bdc774c56b574e04229a66eeae2b9f32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:13:47 -0400 Subject: [PATCH 065/388] Bump org.xerial.snappy:snappy-java from 1.1.10.3 to 1.1.10.4 in /test/fixtures/hdfs-fixture (#10206) * Bump org.xerial.snappy:snappy-java in /test/fixtures/hdfs-fixture Bumps [org.xerial.snappy:snappy-java](https://github.com/xerial/snappy-java) from 1.1.10.3 to 1.1.10.4. - [Release notes](https://github.com/xerial/snappy-java/releases) - [Commits](https://github.com/xerial/snappy-java/compare/v1.1.10.3...v1.1.10.4) --- updated-dependencies: - dependency-name: org.xerial.snappy:snappy-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 3 ++- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4d664e9154bd..1d7e48c3009e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125)) - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752)) - Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) @@ -119,4 +120,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 5217089cf3b4c..4aad96eddaeb5 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -72,5 +72,5 @@ dependencies { exclude group: "com.squareup.okio" } runtimeOnly "com.squareup.okio:okio:3.5.0" - runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.3" + runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.4" } From c44337e92d0cf8c337a36d5b9691d383d4016e42 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 26 Sep 2023 02:11:37 +0800 Subject: [PATCH 066/388] Fix remove ingest processor handing ignore_missing parameter not correctly (#10089) * Fix remove ingest processor handing ignore_missing parameter not correctly Signed-off-by: Gao Binlong * modify change log Signed-off-by: Gao Binlong * Fix test failure and optimize some code Signed-off-by: Gao Binlong * Minor change Signed-off-by: Gao Binlong * Modify test code Signed-off-by: Gao Binlong * Split a test method to two Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../ingest/common/RemoveProcessor.java | 23 +++-- .../ingest/common/RemoveProcessorTests.java | 53 +++++++++-- .../test/ingest/290_remove_processor.yml | 93 +++++++++++++++++++ .../10_pipeline_with_mustache_templates.yml | 2 +- 5 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_remove_processor.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7e48c3009e4..0dbc02a815521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725)) - Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045)) - Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082)) +- Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089)) ### Security diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java index 5da3b6bea7bc2..93a35eef4d396 100644 --- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java +++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java @@ -32,6 +32,7 @@ package org.opensearch.ingest.common; +import org.opensearch.core.common.Strings; import org.opensearch.ingest.AbstractProcessor; import org.opensearch.ingest.ConfigurationUtils; import org.opensearch.ingest.IngestDocument; @@ -66,16 +67,20 @@ public List getFields() { @Override public IngestDocument execute(IngestDocument document) { - if (ignoreMissing) { - fields.forEach(field -> { - String path = document.renderTemplate(field); - if (document.hasField(path)) { - document.removeField(path); + fields.forEach(field -> { + String path = document.renderTemplate(field); + final boolean fieldPathIsNullOrEmpty = Strings.isNullOrEmpty(path); + if (fieldPathIsNullOrEmpty || document.hasField(path) == false) { + if (ignoreMissing) { + return; + } else if (fieldPathIsNullOrEmpty) { + throw new IllegalArgumentException("field path cannot be null nor empty"); + } else { + throw new IllegalArgumentException("field [" + path + "] doesn't exist"); } - }); - } else { - fields.forEach(document::removeField); - } + } + document.removeField(path); + }); return document; } diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorTests.java index cf65236157111..8f729c6a39bbd 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorTests.java @@ -42,7 +42,6 @@ import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class RemoveProcessorTests extends OpenSearchTestCase { @@ -67,12 +66,44 @@ public void testRemoveNonExistingField() throws Exception { config.put("field", fieldName); String processorTag = randomAlphaOfLength(10); Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config); - try { - processor.execute(ingestDocument); - fail("remove field should have failed"); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]")); - } + assertThrows( + "field [" + fieldName + "] doesn't exist", + IllegalArgumentException.class, + () -> { processor.execute(ingestDocument); } + ); + + Map configWithEmptyField = new HashMap<>(); + configWithEmptyField.put("field", ""); + processorTag = randomAlphaOfLength(10); + Processor removeProcessorWithEmptyField = new RemoveProcessor.Factory(TestTemplateService.instance()).create( + null, + processorTag, + null, + configWithEmptyField + ); + assertThrows( + "field path cannot be null nor empty", + IllegalArgumentException.class, + () -> removeProcessorWithEmptyField.execute(ingestDocument) + ); + } + + public void testRemoveEmptyField() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + Map config = new HashMap<>(); + config.put("field", ""); + String processorTag = randomAlphaOfLength(10); + Processor removeProcessorWithEmptyField = new RemoveProcessor.Factory(TestTemplateService.instance()).create( + null, + processorTag, + null, + config + ); + assertThrows( + "field path cannot be null nor empty", + IllegalArgumentException.class, + () -> removeProcessorWithEmptyField.execute(ingestDocument) + ); } public void testIgnoreMissing() throws Exception { @@ -84,5 +115,13 @@ public void testIgnoreMissing() throws Exception { String processorTag = randomAlphaOfLength(10); Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config); processor.execute(ingestDocument); + + // when using template snippet, the resolved field path maybe empty + Map configWithEmptyField = new HashMap<>(); + configWithEmptyField.put("field", ""); + configWithEmptyField.put("ignore_missing", true); + processorTag = randomAlphaOfLength(10); + processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, configWithEmptyField); + processor.execute(ingestDocument); } } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_remove_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_remove_processor.yml new file mode 100644 index 0000000000000..ff5a17136afa2 --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_remove_processor.yml @@ -0,0 +1,93 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + +--- +"Test remove processor with non-existing field and without ignore_missing": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "remove" : { + "field" : "{{unknown}}" + } + } + ] + } + - match: { acknowledged: true } + + - do: + catch: /field path cannot be null nor empty/ + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: { message: "foo bar baz" } + +--- +"Test remove processor with resolved field path doesn't exist": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "remove" : { + "field" : "{{foo}}" + } + } + ] + } + - match: { acknowledged: true } + + - do: + catch: /field \[bar\] doesn\'t exist/ + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: { + message: "foo bar baz", + foo: "bar" + } + +--- +"Test remove processor with non-existing field and ignore_missing": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "remove" : { + "field" : "{{unknown}}", + "ignore_missing" : true + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: { message: "foo bar baz" } + + - do: + get: + index: test + id: 1 + - match: { _source.message: "foo bar baz" } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml index e6a2a3d52e116..c043015281a9a 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml @@ -208,7 +208,7 @@ id: 1 - length: { _source: 2 } - match: { _source.do_nothing: "foo" } - - match: { _source.error: "processor first_processor [remove]: field [field_to_remove] not present as part of path [field_to_remove]" } + - match: { _source.error: "processor first_processor [remove]: field [field_to_remove] doesn't exist" } --- "Test rolling up json object arrays": From 346bcfcdb57f1a966dd2636a38b719776a0a13e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:35:52 -0400 Subject: [PATCH 067/388] Bump codecov/codecov-action from 2 to 3 (#10209) * Bump codecov/codecov-action from 2 to 3 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 2 to 3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v2...v3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/gradle-check.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index bda3bbbc1f125..31578b3fc4660 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -72,7 +72,7 @@ jobs: - name: Upload Coverage Report if: success() - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: files: ./codeCoverage.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dbc02a815521..833d253b19338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752)) - Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126)) - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206)) +- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) From 19fa37c92872bb06d007ba2c1983f389f6185aa2 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Tue, 26 Sep 2023 00:31:53 +0530 Subject: [PATCH 068/388] [Tracing Framework] Add support for SpanKind (#10122) * [Tracing Framework] Add support for SpanKind Signed-off-by: Gagan Juneja * Add CHANGELOG Signed-off-by: Gagan Juneja * Fix test case Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja --------- Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- CHANGELOG.md | 3 +- .../telemetry/tracing/DefaultTracer.java | 42 ++------- .../tracing/SpanCreationContext.java | 81 +++++++++++++++- .../telemetry/tracing/SpanKind.java | 31 +++++++ .../opensearch/telemetry/tracing/Tracer.java | 38 -------- .../telemetry/tracing/TracingTelemetry.java | 8 +- .../telemetry/tracing/noop/NoopTracer.java | 21 ----- .../tracing/runnable/TraceableRunnable.java | 18 +--- .../telemetry/tracing/DefaultTracerTests.java | 93 ++++++++++++------- .../tracing/TraceableRunnableTests.java | 23 +++-- .../tracing/OTelSpanKindConverter.java | 43 +++++++++ .../tracing/OTelTracingTelemetry.java | 24 +++-- .../tracing/OTelSpanKindConverterTests.java | 27 ++++++ .../tracing/OTelTracingTelemetryTests.java | 10 +- .../telemetry/tracing/SpanBuilder.java | 6 +- .../telemetry/tracing/WrappedTracer.java | 26 +----- .../telemetry/tracing/TracerFactoryTests.java | 14 ++- .../telemetry/tracing/WrappedTracerTests.java | 16 ++-- .../test/telemetry/tracing/MockSpan.java | 60 ++++++------ .../tracing/MockTracingContextPropagator.java | 3 +- .../tracing/MockTracingTelemetry.java | 6 +- 21 files changed, 353 insertions(+), 240 deletions(-) create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanKind.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpanKindConverter.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanKindConverterTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 833d253b19338..5105dbb1c1f87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036)) - Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042)) +- [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122)) ### Deprecated @@ -122,4 +123,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java index 5b7795a5647f5..d3c28b3a9cb5e 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java @@ -9,7 +9,6 @@ package org.opensearch.telemetry.tracing; import org.opensearch.common.annotation.InternalApi; -import org.opensearch.telemetry.tracing.attributes.Attributes; import java.io.Closeable; import java.io.IOException; @@ -44,27 +43,13 @@ public DefaultTracer(TracingTelemetry tracingTelemetry, TracerContextStorage> headers) { Optional propagatedSpan = tracingTelemetry.getContextPropagator().extractFromHeaders(headers); - return startSpan( - spanCreationContext.getSpanName(), - propagatedSpan.map(SpanContext::new).orElse(null), - spanCreationContext.getAttributes() - ); + return startSpan(spanCreationContext.parent(propagatedSpan.map(SpanContext::new).orElse(null))); } } diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanCreationContext.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanCreationContext.java index 10cb665e83b01..cbbcfe7a85d57 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanCreationContext.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanCreationContext.java @@ -18,17 +18,74 @@ */ @ExperimentalApi public final class SpanCreationContext { - private final String spanName; - private final Attributes attributes; + private String spanName; + private Attributes attributes; + private SpanKind spanKind = SpanKind.INTERNAL; + private SpanContext parent; /** * Constructor. + */ + private SpanCreationContext() {} + + /** + * Sets the span type to server. + * @return spanCreationContext + */ + public static SpanCreationContext server() { + SpanCreationContext spanCreationContext = new SpanCreationContext(); + spanCreationContext.spanKind = SpanKind.SERVER; + return spanCreationContext; + } + + /** + * Sets the span type to client. + * @return spanCreationContext + */ + public static SpanCreationContext client() { + SpanCreationContext spanCreationContext = new SpanCreationContext(); + spanCreationContext.spanKind = SpanKind.CLIENT; + return spanCreationContext; + } + + /** + * Sets the span type to internal. + * @return spanCreationContext + */ + public static SpanCreationContext internal() { + SpanCreationContext spanCreationContext = new SpanCreationContext(); + spanCreationContext.spanKind = SpanKind.INTERNAL; + return spanCreationContext; + } + + /** + * Sets the span name. * @param spanName span name. - * @param attributes attributes. + * @return spanCreationContext */ - public SpanCreationContext(String spanName, Attributes attributes) { + public SpanCreationContext name(String spanName) { this.spanName = spanName; + return this; + } + + /** + * Sets the span attributes. + * @param attributes attributes. + * @return spanCreationContext + */ + public SpanCreationContext attributes(Attributes attributes) { this.attributes = attributes; + return this; + } + + /** + * Sets the parent for spann + * @param parent parent + * @return spanCreationContext + */ + public SpanCreationContext parent(SpanContext parent) { + this.parent = parent; + return this; } /** @@ -46,4 +103,20 @@ public String getSpanName() { public Attributes getAttributes() { return attributes; } + + /** + * Returns the span kind. + * @return spankind. + */ + public SpanKind getSpanKind() { + return spanKind; + } + + /** + * Returns the parent span + * @return parent. + */ + public SpanContext getParent() { + return parent; + } } diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanKind.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanKind.java new file mode 100644 index 0000000000000..d674bb2c866f2 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanKind.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.common.annotation.PublicApi; + +/** + * Type of Span. + */ +@PublicApi(since = "2.11.0") +public enum SpanKind { + /** + * Span represents the client side code. + */ + CLIENT, + /** + * Span represents the server side code. + */ + SERVER, + + /** + * Span represents the internal calls. This is the default value of a span type. + */ + INTERNAL; +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java index 19ffc68a62df0..49295f417df9e 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java @@ -9,7 +9,6 @@ package org.opensearch.telemetry.tracing; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.telemetry.tracing.attributes.Attributes; import org.opensearch.telemetry.tracing.http.HttpTracer; import java.io.Closeable; @@ -32,34 +31,6 @@ public interface Tracer extends HttpTracer, Closeable { */ Span startSpan(SpanCreationContext context); - /** - * Starts the {@link Span} with given name - * - * @param spanName span name - * @return span, must be closed. - */ - Span startSpan(String spanName); - - /** - * Starts the {@link Span} with given name and attributes. This is required in cases when some attribute based - * decision needs to be made before starting the span. Very useful in the case of Sampling. - * - * @param spanName span name. - * @param attributes attributes to be added. - * @return span, must be closed. - */ - Span startSpan(String spanName, Attributes attributes); - - /** - * Starts the {@link Span} with the given name, parent and attributes. - * - * @param spanName span name. - * @param parentSpan parent span. - * @param attributes attributes to be added. - * @return span, must be closed. - */ - Span startSpan(String spanName, SpanContext parentSpan, Attributes attributes); - /** * Returns the current span. * @return current wrapped span. @@ -74,15 +45,6 @@ public interface Tracer extends HttpTracer, Closeable { */ ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext); - /** - * Start the span and scoped it. This must be used for scenarios where {@link SpanScope} and {@link Span} lifecycles - * are same and ends within the same thread where created. - * @param spanCreationContext span creation context - * @param parentSpan parent span. - * @return scope of the span, must be closed with explicit close or with try-with-resource - */ - ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext, SpanContext parentSpan); - /** * Creates the Span Scope for a current thread. It's mandatory to scope the span just after creation so that it will * automatically manage the attach /detach to the current thread. diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java index e1811e85f8890..38c9104c3ec35 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java @@ -9,7 +9,6 @@ package org.opensearch.telemetry.tracing; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.telemetry.tracing.attributes.Attributes; import java.io.Closeable; @@ -24,12 +23,11 @@ public interface TracingTelemetry extends Closeable { /** * Creates span with provided arguments * - * @param spanName name of the span - * @param parentSpan span's parent span - * @param attributes attributes to be added. + * @param spanCreationContext span creation context. + * @param parentSpan parent span. * @return span instance */ - Span createSpan(String spanName, Span parentSpan, Attributes attributes); + Span createSpan(SpanCreationContext spanCreationContext, Span parentSpan); /** * provides tracing context propagator diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java index d7206a3c6b094..c073e8d3e766f 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java @@ -15,7 +15,6 @@ import org.opensearch.telemetry.tracing.SpanCreationContext; import org.opensearch.telemetry.tracing.SpanScope; import org.opensearch.telemetry.tracing.Tracer; -import org.opensearch.telemetry.tracing.attributes.Attributes; import java.util.List; import java.util.Map; @@ -40,21 +39,6 @@ public Span startSpan(SpanCreationContext context) { return NoopSpan.INSTANCE; } - @Override - public Span startSpan(String spanName) { - return NoopSpan.INSTANCE; - } - - @Override - public Span startSpan(String spanName, Attributes attributes) { - return NoopSpan.INSTANCE; - } - - @Override - public Span startSpan(String spanName, SpanContext parentSpan, Attributes attributes) { - return NoopSpan.INSTANCE; - } - @Override public SpanContext getCurrentSpan() { return new SpanContext(NoopSpan.INSTANCE); @@ -65,11 +49,6 @@ public ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext) { return ScopedSpan.NO_OP; } - @Override - public ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext, SpanContext parentSpan) { - return ScopedSpan.NO_OP; - } - @Override public SpanScope withSpanInScope(Span span) { return SpanScope.NO_OP; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/runnable/TraceableRunnable.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/runnable/TraceableRunnable.java index 4672574e9f4ca..8a61dd70d6d54 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/runnable/TraceableRunnable.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/runnable/TraceableRunnable.java @@ -9,40 +9,32 @@ package org.opensearch.telemetry.tracing.runnable; import org.opensearch.telemetry.tracing.ScopedSpan; -import org.opensearch.telemetry.tracing.SpanContext; import org.opensearch.telemetry.tracing.SpanCreationContext; import org.opensearch.telemetry.tracing.Tracer; -import org.opensearch.telemetry.tracing.attributes.Attributes; /** * Wraps the runnable and add instrumentation to trace the {@link Runnable} */ public class TraceableRunnable implements Runnable { private final Runnable runnable; - private final SpanContext parent; + private final SpanCreationContext spanCreationContext; private final Tracer tracer; - private final String spanName; - private final Attributes attributes; /** * Constructor. * @param tracer tracer - * @param spanName spanName - * @param parent parent Span. - * @param attributes attributes. + * @param spanCreationContext spanCreationContext * @param runnable runnable. */ - public TraceableRunnable(Tracer tracer, String spanName, SpanContext parent, Attributes attributes, Runnable runnable) { + public TraceableRunnable(Tracer tracer, SpanCreationContext spanCreationContext, Runnable runnable) { this.tracer = tracer; - this.spanName = spanName; - this.parent = parent; - this.attributes = attributes; + this.spanCreationContext = spanCreationContext; this.runnable = runnable; } @Override public void run() { - try (ScopedSpan spanScope = tracer.startScopedSpan(new SpanCreationContext(spanName, attributes), parent)) { + try (ScopedSpan spanScope = tracer.startScopedSpan(spanCreationContext)) { runnable.run(); } } diff --git a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java index 48b72e1f673fe..0a717e993cb81 100644 --- a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java +++ b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java @@ -22,7 +22,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -39,6 +38,7 @@ public class DefaultTracerTests extends OpenSearchTestCase { private SpanScope mockSpanScope; private ThreadPool threadPool; private ExecutorService executorService; + private SpanCreationContext spanCreationContext; @Override public void setUp() throws Exception { @@ -58,27 +58,30 @@ public void tearDown() throws Exception { public void testCreateSpan() { DefaultTracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); - defaultTracer.startSpan("span_name"); + defaultTracer.startSpan(spanCreationContext); - assertEquals("span_name", defaultTracer.getCurrentSpan().getSpan().getSpanName()); + String spanName = defaultTracer.getCurrentSpan().getSpan().getSpanName(); + assertEquals("span_name", spanName); } @SuppressWarnings("unchecked") public void testCreateSpanWithAttributesWithMock() { DefaultTracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); Attributes attributes = Attributes.create().addAttribute("name", "value"); - when(mockTracingTelemetry.createSpan(eq("span_name"), eq(mockParentSpan), eq(attributes))).thenReturn(mockSpan); - defaultTracer.startSpan("span_name", attributes); - verify(mockTracingTelemetry).createSpan(eq("span_name"), eq(mockParentSpan), eq(attributes)); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", attributes, mockParentSpan); + when(mockTracingTelemetry.createSpan(eq(spanCreationContext), eq(mockParentSpan))).thenReturn(mockSpan); + defaultTracer.startSpan(spanCreationContext); + verify(mockTracingTelemetry).createSpan(eq(spanCreationContext), eq(mockParentSpan)); } @SuppressWarnings("unchecked") public void testCreateSpanWithAttributesWithParentMock() { DefaultTracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); Attributes attributes = Attributes.create().addAttribute("name", "value"); - when(mockTracingTelemetry.createSpan(eq("span_name"), eq(mockParentSpan), eq(attributes))).thenReturn(mockSpan); - defaultTracer.startSpan("span_name", new SpanContext(mockParentSpan), attributes); - verify(mockTracingTelemetry).createSpan(eq("span_name"), eq(mockParentSpan), eq(attributes)); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", attributes, mockParentSpan); + when(mockTracingTelemetry.createSpan(eq(spanCreationContext), eq(mockParentSpan))).thenReturn(mockSpan); + defaultTracer.startSpan(spanCreationContext); + verify(mockTracingTelemetry).createSpan(eq(spanCreationContext), eq(mockParentSpan)); verify(mockTracerContextStorage, never()).get(TracerContextStorage.CURRENT_SPAN); } @@ -90,11 +93,14 @@ public void testCreateSpanWithAttributes() { new ThreadContextBasedTracerContextStorage(threadContext, tracingTelemetry) ); - Span span = defaultTracer.startSpan( + SpanCreationContext spanCreationContext = buildSpanCreationContext( "span_name", - Attributes.create().addAttribute("key1", 1.0).addAttribute("key2", 2l).addAttribute("key3", true).addAttribute("key4", "key4") + Attributes.create().addAttribute("key1", 1.0).addAttribute("key2", 2l).addAttribute("key3", true).addAttribute("key4", "key4"), + null ); + Span span = defaultTracer.startSpan(spanCreationContext); + assertEquals("span_name", defaultTracer.getCurrentSpan().getSpan().getSpanName()); assertEquals(1.0, ((MockSpan) defaultTracer.getCurrentSpan().getSpan()).getAttribute("key1")); assertEquals(2l, ((MockSpan) defaultTracer.getCurrentSpan().getSpan()).getAttribute("key2")); @@ -110,11 +116,15 @@ public void testCreateSpanWithParent() { new ThreadContextBasedTracerContextStorage(new ThreadContext(Settings.EMPTY), tracingTelemetry) ); - Span span = defaultTracer.startSpan("span_name", null); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", null, null); + + Span span = defaultTracer.startSpan(spanCreationContext, null); SpanContext parentSpan = defaultTracer.getCurrentSpan(); - Span span1 = defaultTracer.startSpan("span_name_1", parentSpan, Attributes.EMPTY); + SpanCreationContext spanCreationContext1 = buildSpanCreationContext("span_name_1", Attributes.EMPTY, parentSpan.getSpan()); + + Span span1 = defaultTracer.startSpan(spanCreationContext1); assertEquals("span_name_1", defaultTracer.getCurrentSpan().getSpan().getSpanName()); assertEquals(parentSpan.getSpan(), defaultTracer.getCurrentSpan().getSpan().getParentSpan()); @@ -126,9 +136,10 @@ public void testCreateSpanWithParent() { public void testCreateSpanWithContext() { DefaultTracer defaultTracer = new DefaultTracer(mockTracingTelemetry, mockTracerContextStorage); Attributes attributes = Attributes.create().addAttribute("name", "value"); - when(mockTracingTelemetry.createSpan(eq("span_name"), eq(mockParentSpan), eq(attributes))).thenReturn(mockSpan); - defaultTracer.startSpan(new SpanCreationContext("span_name", attributes)); - verify(mockTracingTelemetry).createSpan(eq("span_name"), eq(mockParentSpan), eq(attributes)); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", attributes, mockParentSpan); + when(mockTracingTelemetry.createSpan(eq(spanCreationContext), eq(mockParentSpan))).thenReturn(mockSpan); + defaultTracer.startSpan(spanCreationContext); + verify(mockTracingTelemetry).createSpan(eq(spanCreationContext), eq(mockParentSpan)); } public void testCreateSpanWithNullParent() { @@ -139,7 +150,9 @@ public void testCreateSpanWithNullParent() { new ThreadContextBasedTracerContextStorage(threadContext, tracingTelemetry) ); - Span span = defaultTracer.startSpan("span_name", (SpanContext) null, Attributes.EMPTY); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", Attributes.EMPTY, null); + + Span span = defaultTracer.startSpan(spanCreationContext); assertEquals("span_name", defaultTracer.getCurrentSpan().getSpan().getSpanName()); assertEquals(null, defaultTracer.getCurrentSpan().getSpan().getParentSpan()); @@ -154,7 +167,10 @@ public void testEndSpanByClosingScopedSpan() { tracingTelemetry ); DefaultTracer defaultTracer = new DefaultTracer(tracingTelemetry, spanTracerStorage); - ScopedSpan scopedSpan = defaultTracer.startScopedSpan(new SpanCreationContext("span_name", Attributes.EMPTY)); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", Attributes.EMPTY, null); + + ScopedSpan scopedSpan = defaultTracer.startScopedSpan(spanCreationContext); + assertEquals("span_name", defaultTracer.getCurrentSpan().getSpan().getSpanName()); assertEquals(((DefaultScopedSpan) scopedSpan).getSpanScope(), DefaultSpanScope.getCurrentSpanScope()); scopedSpan.close(); @@ -172,8 +188,11 @@ public void testEndSpanByClosingScopedSpanMultiple() { tracingTelemetry ); DefaultTracer defaultTracer = new DefaultTracer(tracingTelemetry, spanTracerStorage); - ScopedSpan scopedSpan = defaultTracer.startScopedSpan(new SpanCreationContext("span_name", Attributes.EMPTY)); - ScopedSpan scopedSpan1 = defaultTracer.startScopedSpan(new SpanCreationContext("span_name_1", Attributes.EMPTY)); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", Attributes.EMPTY, null); + SpanCreationContext spanCreationContext1 = buildSpanCreationContext("span_name_1", Attributes.EMPTY, null); + + ScopedSpan scopedSpan = defaultTracer.startScopedSpan(spanCreationContext); + ScopedSpan scopedSpan1 = defaultTracer.startScopedSpan(spanCreationContext1); assertEquals("span_name_1", defaultTracer.getCurrentSpan().getSpan().getSpanName()); assertEquals(((DefaultScopedSpan) scopedSpan1).getSpanScope(), DefaultSpanScope.getCurrentSpanScope()); @@ -198,7 +217,8 @@ public void testEndSpanByClosingSpanScope() { tracingTelemetry ); DefaultTracer defaultTracer = new DefaultTracer(tracingTelemetry, spanTracerStorage); - Span span = defaultTracer.startSpan(new SpanCreationContext("span_name", Attributes.EMPTY)); + SpanCreationContext spanCreationContext = buildSpanCreationContext("span_name", Attributes.EMPTY, null); + Span span = defaultTracer.startSpan(spanCreationContext); SpanScope spanScope = defaultTracer.withSpanInScope(span); assertEquals("span_name", defaultTracer.getCurrentSpan().getSpan().getSpanName()); assertEquals(spanScope, DefaultSpanScope.getCurrentSpanScope()); @@ -218,8 +238,8 @@ public void testEndSpanByClosingSpanScopeMultiple() { tracingTelemetry ); DefaultTracer defaultTracer = new DefaultTracer(tracingTelemetry, spanTracerStorage); - Span span = defaultTracer.startSpan(new SpanCreationContext("span_name", Attributes.EMPTY)); - Span span1 = defaultTracer.startSpan(new SpanCreationContext("span_name_1", Attributes.EMPTY)); + Span span = defaultTracer.startSpan(buildSpanCreationContext("span_name", Attributes.EMPTY, null)); + Span span1 = defaultTracer.startSpan(buildSpanCreationContext("span_name_1", Attributes.EMPTY, null)); SpanScope spanScope = defaultTracer.withSpanInScope(span); SpanScope spanScope1 = defaultTracer.withSpanInScope(span1); assertEquals("span_name_1", defaultTracer.getCurrentSpan().getSpan().getSpanName()); @@ -261,11 +281,11 @@ public void testSpanAcrossThreads() { CompletableFuture asyncTask = CompletableFuture.runAsync(() -> { // create a span - Span span = defaultTracer.startSpan(new SpanCreationContext("span_name_t_1", Attributes.EMPTY)); + Span span = defaultTracer.startSpan(buildSpanCreationContext("span_name_t_1", Attributes.EMPTY, null)); SpanScope spanScope = defaultTracer.withSpanInScope(span); CompletableFuture asyncTask1 = CompletableFuture.runAsync(() -> { - Span spanT2 = defaultTracer.startSpan(new SpanCreationContext("span_name_t_2", Attributes.EMPTY)); + Span spanT2 = defaultTracer.startSpan(buildSpanCreationContext("span_name_t_2", Attributes.EMPTY, null)); SpanScope spanScopeT2 = defaultTracer.withSpanInScope(spanT2); assertEquals(spanT2, defaultTracer.getCurrentSpan().getSpan()); @@ -289,7 +309,7 @@ public void testSpanCloseOnThread2() { tracingTelemetry ); DefaultTracer defaultTracer = new DefaultTracer(tracingTelemetry, spanTracerStorage); - final Span span = defaultTracer.startSpan(new SpanCreationContext("span_name_t1", Attributes.EMPTY)); + final Span span = defaultTracer.startSpan(buildSpanCreationContext("span_name_t1", Attributes.EMPTY, null)); try (SpanScope spanScope = defaultTracer.withSpanInScope(span)) { CompletableFuture asyncTask = CompletableFuture.runAsync(() -> async(new ActionListener() { @Override @@ -337,16 +357,16 @@ public void testSpanAcrossThreadsMultipleSpans() { CompletableFuture asyncTask = CompletableFuture.runAsync(() -> { // create a parent span - Span parentSpan = defaultTracer.startSpan(new SpanCreationContext("p_span_name", Attributes.EMPTY)); + Span parentSpan = defaultTracer.startSpan(buildSpanCreationContext("p_span_name", Attributes.EMPTY, null)); SpanScope parentSpanScope = defaultTracer.withSpanInScope(parentSpan); // create a span - Span span = defaultTracer.startSpan(new SpanCreationContext("span_name_t_1", Attributes.EMPTY)); + Span span = defaultTracer.startSpan(buildSpanCreationContext("span_name_t_1", Attributes.EMPTY, null)); SpanScope spanScope = defaultTracer.withSpanInScope(span); CompletableFuture asyncTask1 = CompletableFuture.runAsync(() -> { - Span spanT2 = defaultTracer.startSpan(new SpanCreationContext("span_name_t_2", Attributes.EMPTY)); + Span spanT2 = defaultTracer.startSpan(buildSpanCreationContext("span_name_t_2", Attributes.EMPTY, null)); SpanScope spanScopeT2 = defaultTracer.withSpanInScope(spanT2); - Span spanT21 = defaultTracer.startSpan(new SpanCreationContext("span_name_t_2", Attributes.EMPTY)); + Span spanT21 = defaultTracer.startSpan(buildSpanCreationContext("span_name_t_2", Attributes.EMPTY, null)); SpanScope spanScopeT21 = defaultTracer.withSpanInScope(spanT21); assertEquals(spanT21, defaultTracer.getCurrentSpan().getSpan()); spanScopeT21.close(); @@ -390,7 +410,16 @@ private void setupMocks() { when(mockSpan.getParentSpan()).thenReturn(mockParentSpan); when(mockParentSpan.getSpanId()).thenReturn("parent_span_id"); when(mockParentSpan.getTraceId()).thenReturn("trace_id"); - when(mockTracerContextStorage.get(TracerContextStorage.CURRENT_SPAN)).thenReturn(mockParentSpan, mockSpan); - when(mockTracingTelemetry.createSpan(eq("span_name"), eq(mockParentSpan), any(Attributes.class))).thenReturn(mockSpan); + spanCreationContext = buildSpanCreationContext("span_name", Attributes.EMPTY, mockParentSpan); + when(mockTracerContextStorage.get(TracerContextStorage.CURRENT_SPAN)).thenReturn(mockSpan, mockParentSpan); + when(mockTracingTelemetry.createSpan(eq(spanCreationContext), eq(mockParentSpan))).thenReturn(mockSpan); + } + + private SpanCreationContext buildSpanCreationContext(String spanName, Attributes attributes, Span parentSpan) { + SpanCreationContext spanCreationContext = SpanCreationContext.internal().name(spanName).attributes(attributes); + if (parentSpan != null) { + spanCreationContext.parent(new SpanContext(parentSpan)); + } + return spanCreationContext; } } diff --git a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/TraceableRunnableTests.java b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/TraceableRunnableTests.java index a67d9b22ca738..4c4f762653d57 100644 --- a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/TraceableRunnableTests.java +++ b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/TraceableRunnableTests.java @@ -34,9 +34,7 @@ public void testRunnableWithNullParent() throws Exception { final AtomicReference attributeValue = new AtomicReference<>(); TraceableRunnable traceableRunnable = new TraceableRunnable( defaultTracer, - spanName, - null, - Attributes.create().addAttribute("name", "value"), + SpanCreationContext.internal().name(spanName).attributes(Attributes.create().addAttribute("name", "value")), () -> { spanNameCaptured.set(defaultTracer.getCurrentSpan().getSpan().getSpanName()); attributeValue.set((String) ((MockSpan) defaultTracer.getCurrentSpan().getSpan()).getAttribute("name")); @@ -55,14 +53,23 @@ public void testRunnableWithParent() throws Exception { String spanName = "testRunnable"; String parentSpanName = "parentSpan"; DefaultTracer defaultTracer = new DefaultTracer(new MockTracingTelemetry(), contextStorage); - ScopedSpan scopedSpan = defaultTracer.startScopedSpan(new SpanCreationContext(parentSpanName, Attributes.EMPTY)); + ScopedSpan scopedSpan = defaultTracer.startScopedSpan( + SpanCreationContext.internal().name(parentSpanName).attributes(Attributes.EMPTY) + ); SpanContext parentSpanContext = defaultTracer.getCurrentSpan(); AtomicReference currentSpan = new AtomicReference<>(); final AtomicBoolean isRunnableCompleted = new AtomicBoolean(false); - TraceableRunnable traceableRunnable = new TraceableRunnable(defaultTracer, spanName, parentSpanContext, Attributes.EMPTY, () -> { - isRunnableCompleted.set(true); - currentSpan.set(defaultTracer.getCurrentSpan()); - }); + TraceableRunnable traceableRunnable = new TraceableRunnable( + defaultTracer, + SpanCreationContext.internal() + .name(spanName) + .attributes(Attributes.create().addAttribute("name", "value")) + .parent(parentSpanContext), + () -> { + isRunnableCompleted.set(true); + currentSpan.set(defaultTracer.getCurrentSpan()); + } + ); traceableRunnable.run(); assertTrue(isRunnableCompleted.get()); assertEquals(spanName, currentSpan.get().getSpan().getSpanName()); diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpanKindConverter.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpanKindConverter.java new file mode 100644 index 0000000000000..4edb837082126 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpanKindConverter.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import io.opentelemetry.api.trace.SpanKind; + +/** + * Converts {@link org.opensearch.telemetry.tracing.SpanKind} to OTel {@link SpanKind} + */ +final class OTelSpanKindConverter { + + /** + * Constructor. + */ + private OTelSpanKindConverter() {} + + /** + * SpanKind converter. + * @param spanKind span kind. + * @return otel attributes. + */ + static SpanKind convert(org.opensearch.telemetry.tracing.SpanKind spanKind) { + if (spanKind == null) { + return SpanKind.INTERNAL; + } else { + switch (spanKind) { + case CLIENT: + return SpanKind.CLIENT; + case SERVER: + return SpanKind.SERVER; + case INTERNAL: + default: + return SpanKind.INTERNAL; + } + } + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java index ee2f17aabb7f9..53066ad4ad444 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.telemetry.tracing.attributes.Attributes; import java.io.Closeable; import java.io.IOException; @@ -47,8 +46,8 @@ public void close() { } @Override - public Span createSpan(String spanName, Span parentSpan, Attributes attributes) { - return createOtelSpan(spanName, parentSpan, attributes); + public Span createSpan(SpanCreationContext spanCreationContext, Span parentSpan) { + return createOtelSpan(spanCreationContext, parentSpan); } @Override @@ -56,18 +55,29 @@ public TracingContextPropagator getContextPropagator() { return new OTelTracingContextPropagator(openTelemetry); } - private Span createOtelSpan(String spanName, Span parentSpan, Attributes attributes) { - io.opentelemetry.api.trace.Span otelSpan = otelSpan(spanName, parentSpan, OTelAttributesConverter.convert(attributes)); - Span newSpan = new OTelSpan(spanName, otelSpan, parentSpan); + private Span createOtelSpan(SpanCreationContext spanCreationContext, Span parentSpan) { + io.opentelemetry.api.trace.Span otelSpan = otelSpan( + spanCreationContext.getSpanName(), + parentSpan, + OTelAttributesConverter.convert(spanCreationContext.getAttributes()), + OTelSpanKindConverter.convert(spanCreationContext.getSpanKind()) + ); + Span newSpan = new OTelSpan(spanCreationContext.getSpanName(), otelSpan, parentSpan); return newSpan; } - io.opentelemetry.api.trace.Span otelSpan(String spanName, Span parentOTelSpan, io.opentelemetry.api.common.Attributes attributes) { + io.opentelemetry.api.trace.Span otelSpan( + String spanName, + Span parentOTelSpan, + io.opentelemetry.api.common.Attributes attributes, + io.opentelemetry.api.trace.SpanKind spanKind + ) { return parentOTelSpan == null || !(parentOTelSpan instanceof OTelSpan) ? otelTracer.spanBuilder(spanName).setAllAttributes(attributes).startSpan() : otelTracer.spanBuilder(spanName) .setParent(Context.current().with(((OTelSpan) parentOTelSpan).getDelegateSpan())) .setAllAttributes(attributes) + .setSpanKind(spanKind) .startSpan(); } } diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanKindConverterTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanKindConverterTests.java new file mode 100644 index 0000000000000..d07e32d00a92a --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelSpanKindConverterTests.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing; + +import org.opensearch.test.OpenSearchTestCase; + +import io.opentelemetry.api.trace.SpanKind; + +public class OTelSpanKindConverterTests extends OpenSearchTestCase { + + public void testSpanKindNullConverterNull() { + assertEquals(SpanKind.INTERNAL, OTelSpanKindConverter.convert(null)); + } + + public void testSpanKindConverter() { + assertEquals(SpanKind.INTERNAL, OTelSpanKindConverter.convert(org.opensearch.telemetry.tracing.SpanKind.INTERNAL)); + assertEquals(SpanKind.CLIENT, OTelSpanKindConverter.convert(org.opensearch.telemetry.tracing.SpanKind.CLIENT)); + assertEquals(SpanKind.SERVER, OTelSpanKindConverter.convert(org.opensearch.telemetry.tracing.SpanKind.SERVER)); + } + +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java index 3f46cb621a8ec..505756318ff62 100644 --- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java @@ -35,11 +35,11 @@ public void testCreateSpanWithoutParent() { when(mockTracer.spanBuilder("span_name")).thenReturn(mockSpanBuilder); when(mockSpanBuilder.setAllAttributes(any(io.opentelemetry.api.common.Attributes.class))).thenReturn(mockSpanBuilder); when(mockSpanBuilder.startSpan()).thenReturn(mock(io.opentelemetry.api.trace.Span.class)); + when(mockSpanBuilder.setSpanKind(any(io.opentelemetry.api.trace.SpanKind.class))).thenReturn(mockSpanBuilder); Map attributeMap = Collections.singletonMap("name", "value"); Attributes attributes = Attributes.create().addAttribute("name", "value"); TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); - Span span = tracingTelemetry.createSpan("span_name", null, attributes); - + Span span = tracingTelemetry.createSpan(SpanCreationContext.internal().name("span_name").attributes(attributes), null); verify(mockSpanBuilder, never()).setParent(any()); verify(mockSpanBuilder).setAllAttributes(createAttribute(attributes)); assertNull(span.getParentSpan()); @@ -54,12 +54,13 @@ public void testCreateSpanWithParent() { when(mockSpanBuilder.setParent(any())).thenReturn(mockSpanBuilder); when(mockSpanBuilder.setAllAttributes(any(io.opentelemetry.api.common.Attributes.class))).thenReturn(mockSpanBuilder); when(mockSpanBuilder.startSpan()).thenReturn(mock(io.opentelemetry.api.trace.Span.class)); + when(mockSpanBuilder.setSpanKind(any(io.opentelemetry.api.trace.SpanKind.class))).thenReturn(mockSpanBuilder); Span parentSpan = new OTelSpan("parent_span", mock(io.opentelemetry.api.trace.Span.class), null); TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); Attributes attributes = Attributes.create().addAttribute("name", 1l); - Span span = tracingTelemetry.createSpan("span_name", parentSpan, attributes); + Span span = tracingTelemetry.createSpan(SpanCreationContext.internal().name("span_name").attributes(attributes), parentSpan); verify(mockSpanBuilder).setParent(any()); verify(mockSpanBuilder).setAllAttributes(createAttributeLong(attributes)); @@ -77,6 +78,7 @@ public void testCreateSpanWithParentWithMultipleAttributes() { when(mockSpanBuilder.setParent(any())).thenReturn(mockSpanBuilder); when(mockSpanBuilder.setAllAttributes(any(io.opentelemetry.api.common.Attributes.class))).thenReturn(mockSpanBuilder); when(mockSpanBuilder.startSpan()).thenReturn(mock(io.opentelemetry.api.trace.Span.class)); + when(mockSpanBuilder.setSpanKind(any(io.opentelemetry.api.trace.SpanKind.class))).thenReturn(mockSpanBuilder); Span parentSpan = new OTelSpan("parent_span", mock(io.opentelemetry.api.trace.Span.class), null); @@ -86,7 +88,7 @@ public void testCreateSpanWithParentWithMultipleAttributes() { .addAttribute("key2", 2.0) .addAttribute("key3", true) .addAttribute("key4", "key4"); - Span span = tracingTelemetry.createSpan("span_name", parentSpan, attributes); + Span span = tracingTelemetry.createSpan(SpanCreationContext.internal().name("span_name").attributes(attributes), parentSpan); io.opentelemetry.api.common.Attributes otelAttributes = io.opentelemetry.api.common.Attributes.builder() .put("key1", 1l) diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java index d247d48389fc8..a93ce11f374fe 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java @@ -45,7 +45,7 @@ private SpanBuilder() { * @return context. */ public static SpanCreationContext from(HttpRequest request) { - return new SpanCreationContext(createSpanName(request), buildSpanAttributes(request)); + return SpanCreationContext.server().name(createSpanName(request)).attributes(buildSpanAttributes(request)); } /** @@ -54,7 +54,7 @@ public static SpanCreationContext from(HttpRequest request) { * @return context */ public static SpanCreationContext from(RestRequest request) { - return new SpanCreationContext(createSpanName(request), buildSpanAttributes(request)); + return SpanCreationContext.client().name(createSpanName(request)).attributes(buildSpanAttributes(request)); } /** @@ -64,7 +64,7 @@ public static SpanCreationContext from(RestRequest request) { * @return context */ public static SpanCreationContext from(String action, Transport.Connection connection) { - return new SpanCreationContext(createSpanName(action, connection), buildSpanAttributes(action, connection)); + return SpanCreationContext.server().name(createSpanName(action, connection)).attributes(buildSpanAttributes(action, connection)); } private static String createSpanName(HttpRequest httpRequest) { diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java index d8cf39d4a4d09..b2308402379ac 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java @@ -10,7 +10,6 @@ import org.opensearch.common.annotation.InternalApi; import org.opensearch.telemetry.TelemetrySettings; -import org.opensearch.telemetry.tracing.attributes.Attributes; import org.opensearch.telemetry.tracing.noop.NoopTracer; import java.io.IOException; @@ -41,17 +40,7 @@ public WrappedTracer(TelemetrySettings telemetrySettings, Tracer defaultTracer) @Override public Span startSpan(SpanCreationContext context) { - return startSpan(context.getSpanName(), context.getAttributes()); - } - - @Override - public Span startSpan(String spanName) { - return startSpan(spanName, Attributes.EMPTY); - } - - @Override - public Span startSpan(String spanName, Attributes attributes) { - return startSpan(spanName, (SpanContext) null, attributes); + return getDelegateTracer().startSpan(context); } @Override @@ -62,12 +51,7 @@ public SpanContext getCurrentSpan() { @Override public ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext) { - return startScopedSpan(spanCreationContext, null); - } - - @Override - public ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext, SpanContext parentSpan) { - return getDelegateTracer().startScopedSpan(spanCreationContext, parentSpan); + return getDelegateTracer().startScopedSpan(spanCreationContext); } @Override @@ -75,12 +59,6 @@ public SpanScope withSpanInScope(Span span) { return getDelegateTracer().withSpanInScope(span); } - @Override - public Span startSpan(String spanName, SpanContext parentSpan, Attributes attributes) { - Tracer delegateTracer = getDelegateTracer(); - return delegateTracer.startSpan(spanName, parentSpan, attributes); - } - @Override public void close() throws IOException { defaultTracer.close(); diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java index f202be70c9425..b27f888eaf502 100644 --- a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java +++ b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java @@ -48,10 +48,14 @@ public void testGetTracerWithUnavailableTracingTelemetryReturnsNoopTracer() { Tracer tracer = tracerFactory.getTracer(); assertTrue(tracer instanceof NoopTracer); - assertTrue(tracer.startSpan("foo") == NoopSpan.INSTANCE); - assertTrue(tracer.startScopedSpan(new SpanCreationContext("foo", Attributes.EMPTY)) == ScopedSpan.NO_OP); - assertTrue(tracer.startScopedSpan(new SpanCreationContext("foo", Attributes.EMPTY)) == ScopedSpan.NO_OP); - assertTrue(tracer.withSpanInScope(tracer.startSpan("foo")) == SpanScope.NO_OP); + assertTrue(tracer.startSpan(SpanCreationContext.internal().name("foo").attributes(Attributes.EMPTY)) == NoopSpan.INSTANCE); + assertTrue(tracer.startScopedSpan(SpanCreationContext.internal().name("foo").attributes(Attributes.EMPTY)) == ScopedSpan.NO_OP); + assertTrue(tracer.startScopedSpan(SpanCreationContext.internal().name("foo").attributes(Attributes.EMPTY)) == ScopedSpan.NO_OP); + assertTrue( + tracer.withSpanInScope( + tracer.startSpan(SpanCreationContext.internal().name("foo").attributes(Attributes.EMPTY)) + ) == SpanScope.NO_OP + ); } public void testGetTracerWithUnavailableTracingTelemetry() { @@ -64,7 +68,7 @@ public void testGetTracerWithUnavailableTracingTelemetry() { Tracer tracer = tracerFactory.getTracer(); assertTrue(tracer instanceof NoopTracer); - assertTrue(tracer.startScopedSpan(new SpanCreationContext("foo", Attributes.EMPTY)) == ScopedSpan.NO_OP); + assertTrue(tracer.startScopedSpan(SpanCreationContext.internal().name("foo").attributes(Attributes.EMPTY)) == ScopedSpan.NO_OP); } public void testGetTracerWithAvailableTracingTelemetryReturnsWrappedTracer() { diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java index b70fe81d5f9c4..43e0cb8e44439 100644 --- a/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java +++ b/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Set; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -36,9 +35,10 @@ public void testStartSpanWithTracingDisabledInvokesNoopTracer() throws Exception DefaultTracer mockDefaultTracer = mock(DefaultTracer.class); try (WrappedTracer wrappedTracer = new WrappedTracer(telemetrySettings, mockDefaultTracer)) { - wrappedTracer.startSpan("foo"); + SpanCreationContext spanCreationContext = SpanCreationContext.internal().name("foo"); + wrappedTracer.startSpan(spanCreationContext); assertTrue(wrappedTracer.getDelegateTracer() instanceof NoopTracer); - verify(mockDefaultTracer, never()).startSpan("foo"); + verify(mockDefaultTracer, never()).startSpan(SpanCreationContext.internal().name("foo")); } } @@ -48,10 +48,11 @@ public void testStartSpanWithTracingEnabledInvokesDefaultTracer() throws Excepti DefaultTracer mockDefaultTracer = mock(DefaultTracer.class); try (WrappedTracer wrappedTracer = new WrappedTracer(telemetrySettings, mockDefaultTracer)) { - wrappedTracer.startSpan("foo"); + SpanCreationContext spanCreationContext = SpanCreationContext.internal().name("foo"); + wrappedTracer.startSpan(spanCreationContext); assertTrue(wrappedTracer.getDelegateTracer() instanceof DefaultTracer); - verify(mockDefaultTracer).startSpan(eq("foo"), eq((SpanContext) null), any(Attributes.class)); + verify(mockDefaultTracer).startSpan(eq(spanCreationContext)); } } @@ -61,10 +62,11 @@ public void testStartSpanWithTracingEnabledInvokesDefaultTracerWithAttr() throws DefaultTracer mockDefaultTracer = mock(DefaultTracer.class); Attributes attributes = Attributes.create().addAttribute("key", "value"); try (WrappedTracer wrappedTracer = new WrappedTracer(telemetrySettings, mockDefaultTracer)) { - wrappedTracer.startSpan("foo", attributes); + SpanCreationContext spanCreationContext = SpanCreationContext.internal().name("foo"); + wrappedTracer.startSpan(spanCreationContext); assertTrue(wrappedTracer.getDelegateTracer() instanceof DefaultTracer); - verify(mockDefaultTracer).startSpan("foo", (SpanContext) null, attributes); + verify(mockDefaultTracer).startSpan(spanCreationContext); } } diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java index 694a6a2b9e45b..4b661dc0ad0fe 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java @@ -10,6 +10,8 @@ import org.opensearch.telemetry.tracing.AbstractSpan; import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanCreationContext; +import org.opensearch.telemetry.tracing.SpanKind; import org.opensearch.telemetry.tracing.attributes.Attributes; import java.util.HashMap; @@ -29,6 +31,7 @@ public class MockSpan extends AbstractSpan { private boolean hasEnded; private final Long startTime; private Long endTime; + private final SpanKind spanKind; private final Object lock = new Object(); @@ -36,49 +39,43 @@ public class MockSpan extends AbstractSpan { /** * Base Constructor. - * @param spanName Span Name - * @param parentSpan Parent Span - * @param spanProcessor Span Processor - * @param attributes attributes + * + * @param spanCreationContext Span Creation context. + * @param parentSpan Parent Span + * @param spanProcessor Span Processor */ - public MockSpan(String spanName, Span parentSpan, SpanProcessor spanProcessor, Attributes attributes) { + public MockSpan(SpanCreationContext spanCreationContext, Span parentSpan, SpanProcessor spanProcessor) { this( - spanName, + spanCreationContext.getSpanName(), parentSpan, parentSpan != null ? parentSpan.getTraceId() : IdGenerator.generateTraceId(), IdGenerator.generateSpanId(), spanProcessor, - attributes - ); - } - - /** - * Constructor. - * @param spanName span name. - * @param parentSpan parent span name - * @param spanProcessor span processor. - */ - public MockSpan(String spanName, Span parentSpan, SpanProcessor spanProcessor) { - this( - spanName, - parentSpan, - parentSpan != null ? parentSpan.getTraceId() : IdGenerator.generateTraceId(), - IdGenerator.generateSpanId(), - spanProcessor, - Attributes.EMPTY + spanCreationContext.getAttributes(), + SpanKind.INTERNAL ); } /** * Constructor with traceId and SpanIds - * @param spanName Span Name - * @param parentSpan Parent Span - * @param traceId Trace ID - * @param spanId Span ID - * @param spanProcessor Span Processor - * @param attributes attributes + * + * @param spanName Span Name + * @param parentSpan Parent Span + * @param traceId Trace ID + * @param spanId Span ID + * @param spanProcessor Span Processor + * @param attributes attributes + * @param spanKind type of span. */ - public MockSpan(String spanName, Span parentSpan, String traceId, String spanId, SpanProcessor spanProcessor, Attributes attributes) { + public MockSpan( + String spanName, + Span parentSpan, + String traceId, + String spanId, + SpanProcessor spanProcessor, + Attributes attributes, + SpanKind spanKind + ) { super(spanName, parentSpan); this.spanProcessor = spanProcessor; this.metadata = new HashMap<>(); @@ -88,6 +85,7 @@ public MockSpan(String spanName, Span parentSpan, String traceId, String spanId, if (attributes != null) { this.metadata.putAll(attributes.getAttributesMap()); } + this.spanKind = spanKind; } @Override diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java index 7525b4424c243..6d0cd6d0b1290 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingContextPropagator.java @@ -10,6 +10,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanKind; import org.opensearch.telemetry.tracing.TracingContextPropagator; import org.opensearch.telemetry.tracing.attributes.Attributes; @@ -44,7 +45,7 @@ public Optional extract(Map props) { String[] values = value.split(SEPARATOR); String traceId = values[0]; String spanId = values[1]; - return Optional.of(new MockSpan(null, null, traceId, spanId, spanProcessor, Attributes.EMPTY)); + return Optional.of(new MockSpan(null, null, traceId, spanId, spanProcessor, Attributes.EMPTY, SpanKind.INTERNAL)); } else { return Optional.empty(); } diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java index c7f5943719230..39817a208bd18 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockTracingTelemetry.java @@ -9,9 +9,9 @@ package org.opensearch.test.telemetry.tracing; import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanCreationContext; import org.opensearch.telemetry.tracing.TracingContextPropagator; import org.opensearch.telemetry.tracing.TracingTelemetry; -import org.opensearch.telemetry.tracing.attributes.Attributes; import java.util.concurrent.atomic.AtomicBoolean; @@ -29,8 +29,8 @@ public class MockTracingTelemetry implements TracingTelemetry { public MockTracingTelemetry() {} @Override - public Span createSpan(String spanName, Span parentSpan, Attributes attributes) { - Span span = new MockSpan(spanName, parentSpan, spanProcessor, attributes); + public Span createSpan(SpanCreationContext spanCreationContext, Span parentSpan) { + Span span = new MockSpan(spanCreationContext, parentSpan, spanProcessor); if (shutdown.get() == false) { spanProcessor.onStart(span); } From 52aca7fc2267fe3cf1e3aa5bf70597aa20df12aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 16:03:18 -0400 Subject: [PATCH 069/388] Bump com.google.api.grpc:proto-google-common-protos from 2.10.0 to 2.25.0 in /plugins/repository-gcs (#10208) * Bump com.google.api.grpc:proto-google-common-protos Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.10.0 to 2.25.0. - [Release notes](https://github.com/googleapis/sdk-platform-java/releases) - [Changelog](https://github.com/googleapis/sdk-platform-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/sdk-platform-java/compare/v2.10.0...v2.25.0) --- updated-dependencies: - dependency-name: com.google.api.grpc:proto-google-common-protos dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- .../licenses/proto-google-common-protos-2.10.0.jar.sha1 | 1 - .../licenses/proto-google-common-protos-2.25.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/proto-google-common-protos-2.10.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5105dbb1c1f87..32a83a054a4d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752)) - Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126)) - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206)) +- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208)) - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) ### Changed diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 81b55c77941e8..ec040a180876e 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -60,7 +60,7 @@ dependencies { api 'com.google.api-client:google-api-client:2.2.0' - api 'com.google.api.grpc:proto-google-common-protos:2.10.0' + api 'com.google.api.grpc:proto-google-common-protos:2.25.0' api 'com.google.api.grpc:proto-google-iam-v1:0.12.0' api "com.google.auth:google-auth-library-credentials:${versions.google_auth}" diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-2.10.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-2.10.0.jar.sha1 deleted file mode 100644 index bf97707836c70..0000000000000 --- a/plugins/repository-gcs/licenses/proto-google-common-protos-2.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cf5ac081c05682b0eba6659dee55352fde5852e1 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 new file mode 100644 index 0000000000000..b5ef7ee78e794 --- /dev/null +++ b/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 @@ -0,0 +1 @@ +adfa7c3d9b806969db75cf35fe4b286b3b8b1ce0 \ No newline at end of file From f6646fbf134c62898926e6821e81f15e0dea4b51 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Mon, 25 Sep 2023 22:36:16 +0200 Subject: [PATCH 070/388] Bump BouncyCastle to 1.76 (#10219) Signed-off-by: Andrey Pleskach --- CHANGELOG.md | 2 ++ buildSrc/version.properties | 2 +- plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 | 1 - plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 | 1 + .../ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 | 1 - .../ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 | 1 + .../ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 | 1 - .../ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 | 1 + .../ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 | 1 - .../ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 | 1 + 10 files changed, 7 insertions(+), 5 deletions(-) delete mode 100644 plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 create mode 100644 plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a83a054a4d0..d099d61bfe64b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694)) - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666)) - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131)) +- ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) @@ -102,6 +103,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206)) - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208)) - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) +- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 4a31582bb7546..71b3e267700b1 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -48,7 +48,7 @@ reactivestreams = 1.0.4 # when updating this version, you need to ensure compatibility with: # - plugins/ingest-attachment (transitive dependency, check the upstream POM) # - distribution/tools/plugin-cli -bouncycastle=1.75 +bouncycastle=1.76 # test dependencies randomizedrunner = 2.7.1 junit = 4.13.2 diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 deleted file mode 100644 index 9911bb75f9209..0000000000000 --- a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df22e1b6a9f6b218913f5b68dd16641344397fe0 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 new file mode 100644 index 0000000000000..2e96c404bef98 --- /dev/null +++ b/plugins/identity-shiro/licenses/bcprov-jdk15to18-1.76.jar.sha1 @@ -0,0 +1 @@ +0cb53f10290a634808555bc4b34328fdab1001f2 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 deleted file mode 100644 index e6840a9b02b38..0000000000000 --- a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b316bcd094e3917b1ece93a6edbab93f8315fb3b \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 new file mode 100644 index 0000000000000..46010d64015ad --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.76.jar.sha1 @@ -0,0 +1 @@ +23d8bcad6b57912e4633ca9955926ffcdf3c5c71 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 deleted file mode 100644 index 9181b1c3ab1b6..0000000000000 --- a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f16e5252ad7a46d5eaf255231b0a5da307599082 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 new file mode 100644 index 0000000000000..a843d972ac681 --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.76.jar.sha1 @@ -0,0 +1 @@ +3ee440dfa1c557c1cc0c46b5dadf5ef3896ccebb \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 deleted file mode 100644 index 9911bb75f9209..0000000000000 --- a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df22e1b6a9f6b218913f5b68dd16641344397fe0 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 new file mode 100644 index 0000000000000..2e96c404bef98 --- /dev/null +++ b/plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.76.jar.sha1 @@ -0,0 +1 @@ +0cb53f10290a634808555bc4b34328fdab1001f2 \ No newline at end of file From a8969cb11f3a03a9d7f982cb1fab0525ca957535 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 25 Sep 2023 16:10:21 -0700 Subject: [PATCH 071/388] Script Score Query Builder Visit Method (#10198) * Script Score Builder Visit Method Signed-off-by: Varun Jain * ScriptScore Query Builder Visit Method Signed-off-by: Varun Jain * gradlew spotlessApply Signed-off-by: Varun Jain --------- Signed-off-by: Varun Jain --- .../query/functionscore/ScriptScoreQueryBuilder.java | 10 ++++++++++ .../index/query/ScriptScoreQueryBuilderTests.java | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java index e302ebcee4ba7..fe9ad200d44f0 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java @@ -32,6 +32,7 @@ package org.opensearch.index.query.functionscore; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.opensearch.OpenSearchException; import org.opensearch.common.lucene.search.function.ScriptScoreQuery; @@ -45,6 +46,7 @@ import org.opensearch.index.query.InnerHitContextBuilder; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilderVisitor; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.script.ScoreScript; @@ -224,4 +226,12 @@ protected void extractInnerHitBuilders(Map inner InnerHitContextBuilder.extractInnerHits(query(), innerHits); } + @Override + public void visit(QueryBuilderVisitor visitor) { + visitor.accept(this); + if (query != null) { + QueryBuilderVisitor subVisitor = visitor.getChildVisitor(BooleanClause.Occur.MUST); + subVisitor.accept(query); + } + } } diff --git a/server/src/test/java/org/opensearch/index/query/ScriptScoreQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/ScriptScoreQueryBuilderTests.java index 320ff20b2ef5d..29d45fb42ffcc 100644 --- a/server/src/test/java/org/opensearch/index/query/ScriptScoreQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/ScriptScoreQueryBuilderTests.java @@ -43,7 +43,9 @@ import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.hamcrest.CoreMatchers.instanceOf; @@ -140,4 +142,11 @@ public void testDisallowExpensiveQueries() { OpenSearchException e = expectThrows(OpenSearchException.class, () -> queryBuilder.toQuery(queryShardContext)); assertEquals("[script score] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", e.getMessage()); } + + public void testVisit() { + ScriptScoreQueryBuilder scriptQueryBuilder = doCreateTestQueryBuilder(); + List visitedQueries = new ArrayList<>(); + scriptQueryBuilder.visit(createTestVisitor(visitedQueries)); + assertEquals(2, visitedQueries.size()); + } } From 53cffefd22954276a9dc1cdea16ccd5ac2e2bc4c Mon Sep 17 00:00:00 2001 From: Ashish Date: Tue, 26 Sep 2023 15:58:08 +0530 Subject: [PATCH 072/388] Add capability to restrict async durability mode for remote indexes (#10189) Signed-off-by: Ashish Singh --- CHANGELOG.md | 2 +- .../opensearch/remotestore/RemoteStoreIT.java | 81 ++++++++++++++++++ .../metadata/MetadataCreateIndexService.java | 27 +++++- .../MetadataUpdateSettingsService.java | 5 +- .../common/settings/ClusterSettings.java | 3 +- .../opensearch/indices/IndicesService.java | 12 +++ .../MetadataCreateIndexServiceTests.java | 85 +++++++++++++++++++ 7 files changed, 211 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d099d61bfe64b..780db204aa1aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,7 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694)) - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666)) - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131)) -- +- Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 3ccf563941f9c..0aa3a06a108f1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -24,6 +24,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.translog.Translog.Durability; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.plugins.Plugin; @@ -323,6 +324,86 @@ public void testClusterBufferIntervalValidation() { ); } + public void testRequestDurabilityWhenRestrictSettingExplicitFalse() throws ExecutionException, InterruptedException { + // Explicit node settings and request durability + testRestrictSettingFalse(true, Durability.REQUEST); + } + + public void testAsyncDurabilityWhenRestrictSettingExplicitFalse() throws ExecutionException, InterruptedException { + // Explicit node settings and async durability + testRestrictSettingFalse(true, Durability.ASYNC); + } + + public void testRequestDurabilityWhenRestrictSettingImplicitFalse() throws ExecutionException, InterruptedException { + // No node settings and request durability + testRestrictSettingFalse(false, Durability.REQUEST); + } + + public void testAsyncDurabilityWhenRestrictSettingImplicitFalse() throws ExecutionException, InterruptedException { + // No node settings and async durability + testRestrictSettingFalse(false, Durability.ASYNC); + } + + private void testRestrictSettingFalse(boolean setRestrictFalse, Durability durability) throws ExecutionException, InterruptedException { + String clusterManagerName; + if (setRestrictFalse) { + clusterManagerName = internalCluster().startClusterManagerOnlyNode( + Settings.builder().put(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), false).build() + ); + } else { + clusterManagerName = internalCluster().startClusterManagerOnlyNode(); + } + String dataNode = internalCluster().startDataOnlyNodes(1).get(0); + Settings indexSettings = Settings.builder() + .put(indexSettings()) + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), durability) + .build(); + createIndex(INDEX_NAME, indexSettings); + IndexShard indexShard = getIndexShard(dataNode); + assertEquals(durability, indexShard.indexSettings().getTranslogDurability()); + + durability = randomFrom(Durability.values()); + client(clusterManagerName).admin() + .indices() + .updateSettings( + new UpdateSettingsRequest(INDEX_NAME).settings( + Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), durability) + ) + ) + .get(); + assertEquals(durability, indexShard.indexSettings().getTranslogDurability()); + } + + public void testAsyncDurabilityThrowsExceptionWhenRestrictSettingTrue() throws ExecutionException, InterruptedException { + String expectedExceptionMsg = + "index setting [index.translog.durability=async] is not allowed as cluster setting [cluster.remote_store.index.restrict.async-durability=true]"; + String clusterManagerName = internalCluster().startClusterManagerOnlyNode( + Settings.builder().put(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), true).build() + ); + String dataNode = internalCluster().startDataOnlyNodes(1).get(0); + + // Case 1 - Test create index fails + Settings indexSettings = Settings.builder() + .put(indexSettings()) + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Durability.ASYNC) + .build(); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME, indexSettings)); + assertEquals(expectedExceptionMsg, exception.getMessage()); + + // Case 2 - Test update index fails + createIndex(INDEX_NAME); + IndexShard indexShard = getIndexShard(dataNode); + assertEquals(Durability.REQUEST, indexShard.indexSettings().getTranslogDurability()); + exception = assertThrows( + IllegalArgumentException.class, + () -> client(clusterManagerName).admin() + .indices() + .updateSettings(new UpdateSettingsRequest(INDEX_NAME).settings(indexSettings)) + .actionGet() + ); + assertEquals(expectedExceptionMsg, exception.getMessage()); + } + private IndexShard getIndexShard(String dataNode) throws ExecutionException, InterruptedException { String clusterManagerName = internalCluster().getClusterManagerName(); IndicesService indicesService = internalCluster().getInstance(IndicesService.class, dataNode); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index c43353e9e64e0..b2861e566dd4b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -88,6 +88,7 @@ import org.opensearch.index.mapper.MapperService.MergeReason; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexSettingProvider; +import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndexCreationException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.InvalidIndexNameException; @@ -922,6 +923,7 @@ static Settings aggregateIndexSettings( validateTranslogRetentionSettings(indexSettings); validateStoreTypeSettings(indexSettings); validateRefreshIntervalSettings(request.settings(), clusterSettings); + validateTranslogDurabilitySettings(request.settings(), clusterSettings, settings); return indexSettings; } @@ -1491,7 +1493,7 @@ public static void validateTranslogRetentionSettings(Settings indexSettings) { /** * Validates {@code index.refresh_interval} is equal or below the {@code cluster.minimum.index.refresh_interval}. * - * @param requestSettings settings passed in during index create request + * @param requestSettings settings passed in during index create/update request * @param clusterSettings cluster setting */ static void validateRefreshIntervalSettings(Settings requestSettings, ClusterSettings clusterSettings) { @@ -1510,4 +1512,27 @@ static void validateRefreshIntervalSettings(Settings requestSettings, ClusterSet ); } } + + /** + * Validates {@code index.translog.durability} is not async if the {@code cluster.remote_store.index.restrict.async-durability} is set to true. + * + * @param requestSettings settings passed in during index create/update request + * @param clusterSettings cluster setting + */ + static void validateTranslogDurabilitySettings(Settings requestSettings, ClusterSettings clusterSettings, Settings settings) { + if (isRemoteStoreAttributePresent(settings) == false + || IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.exists(requestSettings) == false + || clusterSettings.get(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING) == false) { + return; + } + Translog.Durability durability = IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.get(requestSettings); + if (durability.equals(Translog.Durability.ASYNC)) { + throw new IllegalArgumentException( + "index setting [index.translog.durability=async] is not allowed as cluster setting [" + + IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey() + + "=true]" + ); + } + + } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java index 524980565fe39..7d4c3512ed757 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -73,6 +73,8 @@ import java.util.Set; import static org.opensearch.action.support.ContextPreservingActionListener.wrapPreservingContext; +import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateRefreshIntervalSettings; +import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogDurabilitySettings; import static org.opensearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX; import static org.opensearch.index.IndexSettings.same; @@ -127,7 +129,8 @@ public void updateSettings( .normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX) .build(); - MetadataCreateIndexService.validateRefreshIntervalSettings(normalizedSettings, clusterService.getClusterSettings()); + validateRefreshIntervalSettings(normalizedSettings, clusterService.getClusterSettings()); + validateTranslogDurabilitySettings(normalizedSettings, clusterService.getClusterSettings(), clusterService.getSettings()); Settings.Builder settingsForClosedIndices = Settings.builder(); Settings.Builder settingsForOpenIndices = Settings.builder(); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 22e65b2c04668..032027384f106 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -672,7 +672,8 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING, - IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING + IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, + IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index cf64b886ed523..6f4c51940ce87 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -285,6 +285,18 @@ public class IndicesService extends AbstractLifecycleComponent Property.Dynamic ); + /** + * This setting is used to restrict creation or updation of index where the `index.translog.durability` index setting + * is set as ASYNC if enabled. If disabled, any of the durability mode can be used and switched at any later time from + * one to another. + */ + public static final Setting CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING = Setting.boolSetting( + "cluster.remote_store.index.restrict.async-durability", + false, + Property.NodeScope, + Property.Final + ); + /** * The node's settings. */ diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 8adaae6d230cd..e40826915c848 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -70,6 +70,8 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.translog.Translog; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.InvalidAliasNameException; import org.opensearch.indices.InvalidIndexNameException; import org.opensearch.indices.ShardLimitValidator; @@ -132,8 +134,10 @@ import static org.opensearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; +import static org.opensearch.index.IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.opensearch.node.Node.NODE_ATTRIBUTES; @@ -1725,6 +1729,87 @@ public void testRefreshIntervalValidationFailureWithIndexSetting() { ); } + public void testAnyTranslogDurabilityWhenRestrictSettingFalse() { + // This checks that aggregateIndexSettings works for the case when the cluster setting + // cluster.remote_store.index.restrict.async-durability is false or not set, it allows all types of durability modes + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + Translog.Durability durability = randomFrom(Translog.Durability.values()); + requestSettings.put(INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), durability); + request.settings(requestSettings.build()); + if (randomBoolean()) { + Settings settings = Settings.builder().put(CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), false).build(); + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + } + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + Settings.EMPTY, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + clusterSettings + ); + assertFalse(clusterSettings.get(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING)); + assertEquals(durability, INDEX_TRANSLOG_DURABILITY_SETTING.get(indexSettings)); + } + + public void testAsyncDurabilityThrowsExceptionWhenRestrictSettingTrue() { + // This checks that aggregateIndexSettings works for the case when the cluster setting + // cluster.remote_store.index.restrict.async-durability is false or not set, it allows all types of durability modes + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + requestSettings.put(INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC); + request.settings(requestSettings.build()); + Settings settings = Settings.builder().put(CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), true).build(); + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + Settings.builder().put("node.attr.remote_store.setting", "test").build(), + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + clusterSettings + ) + ); + // verify that the message is as expected + assertEquals( + "index setting [index.translog.durability=async] is not allowed as cluster setting [cluster.remote_store.index.restrict.async-durability=true]", + exception.getMessage() + ); + } + + public void testRequestDurabilityWhenRestrictSettingTrue() { + // This checks that aggregateIndexSettings works for the case when the cluster setting + // cluster.remote_store.index.restrict.async-durability is false or not set, it allows all types of durability modes + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + requestSettings.put(INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST); + request.settings(requestSettings.build()); + Settings settings = Settings.builder().put(CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING.getKey(), true).build(); + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + Settings.EMPTY, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + clusterSettings + ); + assertTrue(clusterSettings.get(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING)); + assertEquals(Translog.Durability.REQUEST, INDEX_TRANSLOG_DURABILITY_SETTING.get(indexSettings)); + } + private IndexTemplateMetadata addMatchingTemplate(Consumer configurator) { IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*"); configurator.accept(builder); From e617da375740065a3878cea093ac8eadaf2a821a Mon Sep 17 00:00:00 2001 From: Ashish Date: Tue, 26 Sep 2023 16:49:41 +0530 Subject: [PATCH 073/388] Enhance debug, trace & info logs for remote store flows (#10182) * Enhance debug, trace & info logs for remote store flows Signed-off-by: Ashish Singh * Add shardId to logger and more logs Signed-off-by: Ashish Singh * Rename unkwown index name to _unknown_ Signed-off-by: Ashish Singh * Empty-Commit Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- .../java/org/opensearch/core/index/Index.java | 1 + .../remote/RemoteSegmentTransferTracker.java | 5 +++ .../CloseableRetryableRefreshListener.java | 5 ++- .../opensearch/index/shard/IndexShard.java | 1 + .../shard/RemoteStoreRefreshListener.java | 18 ++++---- .../opensearch/index/shard/StoreRecovery.java | 2 +- .../store/RemoteSegmentStoreDirectory.java | 32 +++++++++++---- .../RemoteSegmentStoreDirectoryFactory.java | 13 +++--- .../translog/InternalTranslogManager.java | 5 ++- .../index/translog/RemoteFsTranslog.java | 12 ++++-- .../transfer/TranslogTransferManager.java | 3 +- .../LocalStorePeerRecoverySourceHandler.java | 3 ++ .../recovery/PeerRecoveryTargetService.java | 6 +-- .../recovery/RecoverySourceHandler.java | 41 ++++++++++++++----- .../RemoteStorePeerRecoverySourceHandler.java | 2 + .../RemoteStoreReplicationSource.java | 4 +- .../SegmentReplicationTargetService.java | 11 ++--- .../checkpoint/PublishCheckpointAction.java | 4 +- .../blobstore/BlobStoreRepository.java | 13 +++++- .../RemoteStoreRefreshListenerTests.java | 3 +- .../RemoteSegmentStoreDirectoryTests.java | 20 +++++---- .../index/shard/IndexShardTestCase.java | 2 +- 22 files changed, 142 insertions(+), 64 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/index/Index.java b/libs/core/src/main/java/org/opensearch/core/index/Index.java index 77cc628213df9..b5ee482bcd952 100644 --- a/libs/core/src/main/java/org/opensearch/core/index/Index.java +++ b/libs/core/src/main/java/org/opensearch/core/index/Index.java @@ -59,6 +59,7 @@ public class Index implements Writeable, ToXContentObject { public static final Index[] EMPTY_ARRAY = new Index[0]; private static final String INDEX_UUID_KEY = "index_uuid"; private static final String INDEX_NAME_KEY = "index_name"; + public static final String UNKNOWN_INDEX_NAME = "_unknown_"; private static final ObjectParser INDEX_PARSER = new ObjectParser<>("index", Builder::new); static { diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java index d18462cd65fc1..05081180bb179 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java @@ -294,6 +294,11 @@ public void updateLatestLocalFileNameLengthMap( Collection segmentFiles, CheckedFunction fileSizeFunction ) { + logger.debug( + "segmentFilesPostRefresh={} latestLocalFileNamesBeforeMapUpdate={}", + segmentFiles, + latestLocalFileNameLengthMap.keySet() + ); // Update the map segmentFiles.stream() .filter(file -> EXCLUDE_FILES.contains(file) == false) diff --git a/server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java index 9bc105bf13f0a..3ee74e5267718 100644 --- a/server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/CloseableRetryableRefreshListener.java @@ -104,6 +104,7 @@ protected TimeValue getNextRetryInterval() { private void scheduleRetry(TimeValue interval, String retryThreadPoolName, boolean didRefresh) { // If the underlying listener has closed, then we do not allow even the retry to be scheduled if (closed.get() || isRetryEnabled() == false) { + getLogger().debug("skip retry on closed={} isRetryEnabled={}", closed.get(), isRetryEnabled()); return; } @@ -112,6 +113,7 @@ private void scheduleRetry(TimeValue interval, String retryThreadPoolName, boole // If the retryScheduled is already true, then we return from here itself. If not, then we proceed with scheduling // the retry. if (retryScheduled.getAndSet(true)) { + getLogger().debug("skip retry on retryScheduled=true"); return; } @@ -188,7 +190,7 @@ public final void close() throws IOException { if (semaphore.tryAcquire(TOTAL_PERMITS, 10, TimeUnit.MINUTES)) { boolean result = closed.compareAndSet(false, true); assert result && semaphore.availablePermits() == 0; - getLogger().info("Closed"); + getLogger().info("All permits are acquired and refresh listener is closed"); } else { throw new TimeoutException("timeout while closing gated refresh listener"); } @@ -200,7 +202,6 @@ public final void close() throws IOException { protected abstract Logger getLogger(); // Visible for testing - /** * Returns if the retry is scheduled or not. * diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 6ab2110370f52..bbb7d3cf5f30f 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -3508,6 +3508,7 @@ public void startRecovery( // } // }} // } + logger.debug("startRecovery type={}", recoveryState.getRecoverySource().getType()); assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource()); switch (recoveryState.getRecoverySource().getType()) { case EMPTY_STORE: diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 4e035af9bbc5e..695c01367171a 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -127,6 +127,7 @@ protected void runAfterRefreshExactlyOnce(boolean didRefresh) { segmentTracker.updateLocalRefreshTimeAndSeqNo(); try { if (this.primaryTerm != indexShard.getOperationPrimaryTerm()) { + logger.debug("primaryTerm update from={} to={}", primaryTerm, indexShard.getOperationPrimaryTerm()); this.primaryTerm = indexShard.getOperationPrimaryTerm(); this.remoteDirectory.init(); } @@ -150,8 +151,6 @@ protected void runAfterRefreshExactlyOnce(boolean didRefresh) { @Override protected boolean performAfterRefreshWithPermit(boolean didRefresh) { boolean successful; - // The third condition exists for uploading the zero state segments where the refresh has not changed the reader reference, but it - // is important to upload the zero state segments so that the restore does not break. if (shouldSync(didRefresh)) { successful = syncSegments(); } else { @@ -161,6 +160,8 @@ protected boolean performAfterRefreshWithPermit(boolean didRefresh) { } private boolean shouldSync(boolean didRefresh) { + // The third condition exists for uploading the zero state segments where the refresh has not changed the reader reference, but it + // is important to upload the zero state segments so that the restore does not break. return this.primaryTerm != indexShard.getOperationPrimaryTerm() || didRefresh || remoteDirectory.getSegmentsUploadedToRemoteStore().isEmpty(); @@ -168,7 +169,7 @@ private boolean shouldSync(boolean didRefresh) { private boolean syncSegments() { if (indexShard.getReplicationTracker().isPrimaryMode() == false || indexShard.state() == IndexShardState.CLOSED) { - logger.trace( + logger.debug( "Skipped syncing segments with primaryMode={} indexShardState={}", indexShard.getReplicationTracker().isPrimaryMode(), indexShard.state() @@ -178,10 +179,7 @@ private boolean syncSegments() { // primaryMode to true. Due to this, the refresh that is triggered post replay of translog will not go through // if following condition does not exist. The segments created as part of translog replay will not be present // in the remote store. - if (indexShard.state() == IndexShardState.STARTED && indexShard.getEngine() instanceof InternalEngine) { - return false; - } - return true; + return indexShard.state() != IndexShardState.STARTED || !(indexShard.getEngine() instanceof InternalEngine); } ReplicationCheckpoint checkpoint = indexShard.getLatestReplicationCheckpoint(); beforeSegmentsSync(); @@ -217,8 +215,10 @@ private boolean syncSegments() { @Override public void onResponse(Void unused) { try { + logger.debug("New segments upload successful"); // Start metadata file upload uploadMetadata(localSegmentsPostRefresh, segmentInfos, checkpoint); + logger.debug("Metadata upload successful"); clearStaleFilesFromLocalSegmentChecksumMap(localSegmentsPostRefresh); onSuccessfulSegmentsSync( refreshTimeMs, @@ -262,6 +262,7 @@ public void onFailure(Exception e) { updateFinalStatusInSegmentTracker(successful.get(), bytesBeforeUpload, startTimeInNS); // If there are failures in uploading segments, then we should retry as search idle can lead to // refresh not occurring until write happens. + logger.debug("syncSegments runStatus={}", successful.get()); return successful.get(); } @@ -300,6 +301,7 @@ private void onSuccessfulSegmentsSync( indexShard.getEngine().translogManager().setMinSeqNoToKeep(lastRefreshedCheckpoint + 1); // Publishing the new checkpoint which is used for remote store + segrep indexes checkpointPublisher.publish(indexShard, checkpoint); + logger.debug("onSuccessfulSegmentsSync lastRefreshedCheckpoint={} checkpoint={}", lastRefreshedCheckpoint, checkpoint); } /** @@ -352,10 +354,12 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se private void uploadNewSegments(Collection localSegmentsPostRefresh, ActionListener listener) { Collection filteredFiles = localSegmentsPostRefresh.stream().filter(file -> !skipUpload(file)).collect(Collectors.toList()); if (filteredFiles.size() == 0) { + logger.debug("No new segments to upload in uploadNewSegments"); listener.onResponse(null); return; } + logger.debug("Effective new segments files to upload {}", filteredFiles); ActionListener> mappedListener = ActionListener.map(listener, resp -> null); GroupedActionListener batchUploadListener = new GroupedActionListener<>(mappedListener, filteredFiles.size()); diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 6d675b709e05b..f429d94f7f96c 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -399,7 +399,7 @@ void recoverFromSnapshotAndRemoteStore( RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( remoteStoreRepository, indexUUID, - String.valueOf(shardId.id()) + shardId ); indexShard.syncSegmentsFromGivenRemoteSegmentStore(true, sourceRemoteDirectory, primaryTerm, commitGeneration); final Store store = indexShard.store(); diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index a68fed5e302bd..b23d2d7d0a3f8 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -26,8 +26,10 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.io.VersionedCodecStreamWrapper; +import org.opensearch.common.logging.Loggers; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.store.lockmanager.FileLockInfo; import org.opensearch.index.store.lockmanager.RemoteStoreCommitLevelLockManager; @@ -42,6 +44,7 @@ import java.io.InputStream; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -99,7 +102,9 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement RemoteSegmentMetadata.METADATA_CODEC ); - private static final Logger logger = LogManager.getLogger(RemoteSegmentStoreDirectory.class); + private static final Logger staticLogger = LogManager.getLogger(RemoteSegmentStoreDirectory.class); + + private final Logger logger; /** * AtomicBoolean that ensures only one staleCommitDeletion activity is scheduled at a time. @@ -113,13 +118,15 @@ public RemoteSegmentStoreDirectory( RemoteDirectory remoteDataDirectory, RemoteDirectory remoteMetadataDirectory, RemoteStoreLockManager mdLockManager, - ThreadPool threadPool + ThreadPool threadPool, + ShardId shardId ) throws IOException { super(remoteDataDirectory); this.remoteDataDirectory = remoteDataDirectory; this.remoteMetadataDirectory = remoteMetadataDirectory; this.mdLockManager = mdLockManager; this.threadPool = threadPool; + this.logger = Loggers.getLogger(getClass(), shardId); init(); } @@ -132,12 +139,14 @@ public RemoteSegmentStoreDirectory( * @throws IOException if there were any failures in reading the metadata file */ public RemoteSegmentMetadata init() throws IOException { + logger.debug("Start initialisation of remote segment metadata"); RemoteSegmentMetadata remoteSegmentMetadata = readLatestMetadataFile(); if (remoteSegmentMetadata != null) { this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(remoteSegmentMetadata.getMetadata()); } else { this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(); } + logger.debug("Initialisation of remote segment metadata completed"); return remoteSegmentMetadata; } @@ -250,7 +259,7 @@ public static UploadedSegmentMetadata fromString(String uploadedFilename) { String[] values = uploadedFilename.split(SEPARATOR); UploadedSegmentMetadata metadata = new UploadedSegmentMetadata(values[0], values[1], values[2], Long.parseLong(values[3])); if (values.length < 5) { - logger.error("Lucene version is missing for UploadedSegmentMetadata: " + uploadedFilename); + staticLogger.error("Lucene version is missing for UploadedSegmentMetadata: " + uploadedFilename); } metadata.setWrittenByMajor(Integer.parseInt(values[4])); @@ -678,7 +687,7 @@ private Map getSegmentToLuceneVersion(Collection segmen */ private void tryAndDeleteLocalFile(String filename, Directory directory) { try { - logger.trace("Deleting file: " + filename); + logger.debug("Deleting file: " + filename); directory.deleteFile(filename); } catch (NoSuchFileException | FileNotFoundException e) { logger.trace("Exception while deleting. Missing file : " + filename, e); @@ -728,7 +737,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException Integer.MAX_VALUE ); if (sortedMetadataFileList.size() <= lastNMetadataFilesToKeep) { - logger.trace( + logger.debug( "Number of commits in remote segment store={}, lastNMetadataFilesToKeep={}", sortedMetadataFileList.size(), lastNMetadataFilesToKeep @@ -756,6 +765,11 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException }).collect(Collectors.toList()); sortedMetadataFileList.removeAll(metadataFilesToBeDeleted); + logger.debug( + "metadataFilesEligibleToDelete={} metadataFilesToBeDeleted={}", + metadataFilesEligibleToDelete, + metadataFilesEligibleToDelete + ); Map activeSegmentFilesMetadataMap = new HashMap<>(); Set activeSegmentRemoteFilenames = new HashSet<>(); @@ -773,9 +787,11 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException .map(metadata -> metadata.uploadedFilename) .collect(Collectors.toSet()); AtomicBoolean deletionSuccessful = new AtomicBoolean(true); + List nonActiveDeletedSegmentFiles = new ArrayList<>(); staleSegmentRemoteFilenames.stream().filter(file -> !activeSegmentRemoteFilenames.contains(file)).forEach(file -> { try { remoteDataDirectory.deleteFile(file); + nonActiveDeletedSegmentFiles.add(file); if (!activeSegmentFilesMetadataMap.containsKey(getLocalSegmentFilename(file))) { segmentsUploadedToRemoteStore.remove(getLocalSegmentFilename(file)); } @@ -790,8 +806,9 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException ); } }); + logger.debug("nonActiveDeletedSegmentFiles={}", nonActiveDeletedSegmentFiles); if (deletionSuccessful.get()) { - logger.trace("Deleting stale metadata file {} from remote segment store", metadataFile); + logger.debug("Deleting stale metadata file {} from remote segment store", metadataFile); remoteMetadataDirectory.deleteFile(metadataFile); } } @@ -842,7 +859,7 @@ private boolean deleteIfEmpty() throws IOException { 1 ); if (metadataFiles.size() != 0) { - logger.info("Remote directory still has files , not deleting the path"); + logger.info("Remote directory still has files, not deleting the path"); return false; } @@ -858,6 +875,7 @@ private boolean deleteIfEmpty() throws IOException { return true; } + @Override public void close() throws IOException { deleteStaleSegmentsAsync(0, ActionListener.wrap(r -> deleteIfEmpty(), e -> logger.error("Failed to cleanup remote directory"))); } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 31b49f6813ad2..490b07e441702 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -10,6 +10,7 @@ import org.apache.lucene.store.Directory; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; @@ -45,17 +46,15 @@ public RemoteSegmentStoreDirectoryFactory(Supplier reposito public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throws IOException { String repositoryName = indexSettings.getRemoteStoreRepository(); String indexUUID = indexSettings.getIndex().getUUID(); - String shardId = String.valueOf(path.getShardId().getId()); - - return newDirectory(repositoryName, indexUUID, shardId); + return newDirectory(repositoryName, indexUUID, path.getShardId()); } - public Directory newDirectory(String repositoryName, String indexUUID, String shardId) throws IOException { + public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId) throws IOException { try (Repository repository = repositoriesService.get().repository(repositoryName)) { assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobStoreRepository blobStoreRepository = ((BlobStoreRepository) repository); BlobPath commonBlobPath = blobStoreRepository.basePath(); - commonBlobPath = commonBlobPath.add(indexUUID).add(shardId).add(SEGMENTS); + commonBlobPath = commonBlobPath.add(indexUUID).add(String.valueOf(shardId.id())).add(SEGMENTS); RemoteDirectory dataDirectory = new RemoteDirectory( blobStoreRepository.blobStore().blobContainer(commonBlobPath.add("data")), @@ -69,10 +68,10 @@ public Directory newDirectory(String repositoryName, String indexUUID, String sh repositoriesService.get(), repositoryName, indexUUID, - shardId + String.valueOf(shardId.id()) ); - return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool, shardId); } catch (RepositoryMissingException e) { throw new IllegalArgumentException("Repository should be created before creating index with remote_store enabled setting", e); } diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java index 6b51cb17dcc41..4bc9a711894b7 100644 --- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java @@ -8,10 +8,10 @@ package org.opensearch.index.translog; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.store.AlreadyClosedException; +import org.opensearch.common.logging.Loggers; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.index.shard.ShardId; @@ -43,7 +43,7 @@ public class InternalTranslogManager implements TranslogManager, Closeable { private final AtomicBoolean pendingTranslogRecovery = new AtomicBoolean(false); private final TranslogEventListener translogEventListener; private final Supplier localCheckpointTrackerSupplier; - private static final Logger logger = LogManager.getLogger(InternalTranslogManager.class); + private final Logger logger; public InternalTranslogManager( TranslogConfig translogConfig, @@ -76,6 +76,7 @@ public InternalTranslogManager( assert pendingTranslogRecovery.get() == false : "translog recovery can't be pending before we set it"; // don't allow commits until we are done with recovering pendingTranslogRecovery.set(true); + this.logger = Loggers.getLogger(getClass(), shardId); } /** diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 23903e5a9d110..004d1bfdca36d 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -181,11 +181,12 @@ static void download(TranslogTransferManager translogTransferManager, Path locat ex = e; } } + logger.debug("Exhausted all download retries during translog/checkpoint file download"); throw ex; } static private void downloadOnce(TranslogTransferManager translogTransferManager, Path location, Logger logger) throws IOException { - logger.trace("Downloading translog files from remote"); + logger.debug("Downloading translog files from remote"); RemoteTranslogTransferTracker statsTracker = translogTransferManager.getRemoteTranslogTransferTracker(); long prevDownloadBytesSucceeded = statsTracker.getDownloadBytesSucceeded(); long prevDownloadTimeInMillis = statsTracker.getTotalDownloadTimeInMillis(); @@ -205,6 +206,11 @@ static private void downloadOnce(TranslogTransferManager translogTransferManager String generation = Long.toString(i); translogTransferManager.downloadTranslog(generationToPrimaryTermMapper.get(generation), generation, location); } + logger.info( + "Downloaded translog and checkpoint files from={} to={}", + translogMetadata.getMinTranslogGeneration(), + translogMetadata.getGeneration() + ); statsTracker.recordDownloadStats(prevDownloadBytesSucceeded, prevDownloadTimeInMillis); @@ -215,7 +221,7 @@ static private void downloadOnce(TranslogTransferManager translogTransferManager location.resolve(Translog.CHECKPOINT_FILE_NAME) ); } - logger.trace("Downloaded translog files from remote"); + logger.debug("downloadOnce execution completed"); } public static TranslogTransferManager buildTranslogTransferManager( @@ -310,7 +316,7 @@ private boolean upload(Long primaryTerm, Long generation) throws IOException { // primary, the engine is reset to InternalEngine which also initialises the RemoteFsTranslog which in turns // downloads all the translogs from remote store and does a flush before the relocation finishes. if (primaryModeSupplier.getAsBoolean() == false) { - logger.trace("skipped uploading translog for {} {}", primaryTerm, generation); + logger.debug("skipped uploading translog for {} {}", primaryTerm, generation); // NO-OP return true; } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index fd4936603671c..d7e50cdcf32e4 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -227,7 +227,7 @@ private void captureStatsOnUploadFailure() { } public boolean downloadTranslog(String primaryTerm, String generation, Path location) throws IOException { - logger.info( + logger.trace( "Downloading translog files with: Primary Term = {}, Generation = {}, Location = {}", primaryTerm, generation, @@ -289,6 +289,7 @@ public TranslogTransferMetadata readMetadata() throws IOException { exceptionSetOnce.set(e); } finally { remoteTranslogTransferTracker.addDownloadTimeInMillis((System.nanoTime() - downloadStartTime) / 1_000_000L); + logger.debug("translogMetadataDownloadStatus={}", downloadStatus); if (downloadStatus) { remoteTranslogTransferTracker.addDownloadBytesSucceeded(bytesToRead); } diff --git a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java index a915046c381fc..ac6b2e6b77d18 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java @@ -161,6 +161,7 @@ && isTargetSameHistory() }, shardId + " removing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger); deleteRetentionLeaseStep.whenComplete(ignored -> { + logger.debug("deleteRetentionLeaseStep completed"); assert Transports.assertNotTransportThread(this + "[phase1]"); phase1(wrappedSafeCommit.get(), startingSeqNo, () -> estimateNumOps, sendFileStep, false); }, onFailure); @@ -172,12 +173,14 @@ && isTargetSameHistory() assert startingSeqNo >= 0 : "startingSeqNo must be non negative. got: " + startingSeqNo; sendFileStep.whenComplete(r -> { + logger.debug("sendFileStep completed"); assert Transports.assertNotTransportThread(this + "[prepareTargetForTranslog]"); // For a sequence based recovery, the target can keep its local translog prepareTargetForTranslog(countNumberOfHistoryOperations(startingSeqNo), prepareEngineStep); }, onFailure); prepareEngineStep.whenComplete(prepareEngineTime -> { + logger.debug("prepareEngineStep completed"); assert Transports.assertNotTransportThread(this + "[phase2]"); /* * add shard to replication group (shard will receive replication requests from this point on) now that engine is open. diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index df1589b3f29b9..1ea32f4e355e3 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -264,7 +264,7 @@ private void doRecovery(final long recoveryId, final StartRecoveryRequest preExi actionName = PeerRecoverySourceService.Actions.START_RECOVERY; } catch (final Exception e) { // this will be logged as warning later on... - logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e); + logger.debug("unexpected error while preparing shard for peer recovery, failing recovery", e); onGoingRecoveries.fail( recoveryId, new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e), @@ -272,12 +272,12 @@ private void doRecovery(final long recoveryId, final StartRecoveryRequest preExi ); return; } - logger.trace("{} starting recovery from {}", startRequest.shardId(), startRequest.sourceNode()); + logger.debug("{} starting recovery from {}", startRequest.shardId(), startRequest.sourceNode()); } else { startRequest = preExistingRequest; requestToSend = new ReestablishRecoveryRequest(recoveryId, startRequest.shardId(), startRequest.targetAllocationId()); actionName = PeerRecoverySourceService.Actions.REESTABLISH_RECOVERY; - logger.trace("{} reestablishing recovery from {}", startRequest.shardId(), startRequest.sourceNode()); + logger.debug("{} reestablishing recovery from {}", startRequest.shardId(), startRequest.sourceNode()); } } transportService.sendRequest( diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 6d7972d79a96c..7996c48b2b04b 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -201,9 +201,13 @@ protected void finalizeStepAndCompleteFuture( final StepListener finalizeStep = new StepListener<>(); // Recovery target can trim all operations >= startingSeqNo as we have sent all these operations in the phase 2 final long trimAboveSeqNo = startingSeqNo - 1; - sendSnapshotStep.whenComplete(r -> finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep), onFailure); + sendSnapshotStep.whenComplete(r -> { + logger.debug("sendSnapshotStep completed"); + finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep); + }, onFailure); finalizeStep.whenComplete(r -> { + logger.debug("finalizeStep completed"); final long phase1ThrottlingWaitTime = 0L; // TODO: return the actual throttle time final SendSnapshotResult sendSnapshotResult = sendSnapshotStep.result(); final SendFileResult sendFileResult = sendFileStep.result(); @@ -233,7 +237,10 @@ protected void onSendFileStepComplete( GatedCloseable wrappedSafeCommit, Releasable releaseStore ) { - sendFileStep.whenComplete(r -> IOUtils.close(wrappedSafeCommit, releaseStore), e -> { + sendFileStep.whenComplete(r -> { + logger.debug("sendFileStep completed"); + IOUtils.close(wrappedSafeCommit, releaseStore); + }, e -> { try { IOUtils.close(wrappedSafeCommit, releaseStore); } catch (final IOException ex) { @@ -445,16 +452,22 @@ void phase1( sendFileInfoStep ); - sendFileInfoStep.whenComplete( - r -> sendFiles(store, phase1Files.toArray(new StoreFileMetadata[0]), translogOps, sendFilesStep), - listener::onFailure - ); + sendFileInfoStep.whenComplete(r -> { + logger.debug("sendFileInfoStep completed"); + sendFiles(store, phase1Files.toArray(new StoreFileMetadata[0]), translogOps, sendFilesStep); + }, listener::onFailure); // When doing peer recovery of remote store enabled replica, retention leases are not required. if (skipCreateRetentionLeaseStep) { - sendFilesStep.whenComplete(r -> createRetentionLeaseStep.onResponse(null), listener::onFailure); + sendFilesStep.whenComplete(r -> { + logger.debug("sendFilesStep completed"); + createRetentionLeaseStep.onResponse(null); + }, listener::onFailure); } else { - sendFilesStep.whenComplete(r -> createRetentionLease(startingSeqNo, createRetentionLeaseStep), listener::onFailure); + sendFilesStep.whenComplete(r -> { + logger.debug("sendFilesStep completed"); + createRetentionLease(startingSeqNo, createRetentionLeaseStep); + }, listener::onFailure); } createRetentionLeaseStep.whenComplete(retentionLease -> { @@ -471,6 +484,7 @@ void phase1( final long totalSize = totalSizeInBytes; final long existingTotalSize = existingTotalSizeInBytes; cleanFilesStep.whenComplete(r -> { + logger.debug("cleanFilesStep completed"); final TimeValue took = stopWatch.totalTime(); logger.trace("recovery [phase1]: took [{}]", took); listener.onResponse( @@ -541,7 +555,10 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, cloneRetentionLeaseStep, false) ); logger.trace("cloned primary's retention lease as [{}]", clonedLease); - cloneRetentionLeaseStep.whenComplete(rr -> listener.onResponse(clonedLease), listener::onFailure); + cloneRetentionLeaseStep.whenComplete(rr -> { + logger.debug("cloneRetentionLeaseStep completed"); + listener.onResponse(clonedLease); + }, listener::onFailure); } catch (RetentionLeaseNotFoundException e) { // it's possible that the primary has no retention lease yet if we are doing a rolling upgrade from a version before // 7.4, and in that case we just create a lease using the local checkpoint of the safe commit which we're using for @@ -554,7 +571,10 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, addRetentionLeaseStep, false) ); - addRetentionLeaseStep.whenComplete(rr -> listener.onResponse(newLease), listener::onFailure); + addRetentionLeaseStep.whenComplete(rr -> { + logger.debug("addRetentionLeaseStep completed"); + listener.onResponse(newLease); + }, listener::onFailure); logger.trace("created retention lease with estimated checkpoint of [{}]", estimatedGlobalCheckpoint); } }, shardId + " establishing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger); @@ -810,6 +830,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis cancellableThreads.checkForCancel(); recoveryTarget.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, finalizeListener); finalizeListener.whenComplete(r -> { + logger.debug("finalizeListenerStep completed"); RunUnderPrimaryPermit.run( () -> shard.updateGlobalCheckpointForShard(request.targetAllocationId(), globalCheckpoint), shardId + " updating " + request.targetAllocationId() + "'s global checkpoint", diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java index 5d6c0eb3bae05..66c7a3b48f28f 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java @@ -80,12 +80,14 @@ protected void innerRecoveryToTarget(ActionListener listener, assert startingSeqNo >= 0 : "startingSeqNo must be non negative. got: " + startingSeqNo; sendFileStep.whenComplete(r -> { + logger.debug("sendFileStep completed"); assert Transports.assertNotTransportThread(this + "[prepareTargetForTranslog]"); // For a sequence based recovery, the target can keep its local translog prepareTargetForTranslog(0, prepareEngineStep); }, onFailure); prepareEngineStep.whenComplete(prepareEngineTime -> { + logger.debug("prepareEngineStep completed"); assert Transports.assertNotTransportThread(this + "[phase2]"); RunUnderPrimaryPermit.run( () -> shard.initiateTracking(request.targetAllocationId()), diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java index 64aacd68f2490..aeb690465905f 100644 --- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java @@ -105,7 +105,7 @@ public void getSegmentFiles( listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); return; } - logger.trace("Downloading segments files from remote store {}", filesToFetch); + logger.debug("Downloading segment files from remote store {}", filesToFetch); RemoteSegmentMetadata remoteSegmentMetadata = remoteDirectory.readLatestMetadataFile(); List toDownloadSegments = new ArrayList<>(); @@ -122,7 +122,7 @@ public void getSegmentFiles( toDownloadSegments.add(fileMetadata); } downloadSegments(storeDirectory, remoteDirectory, toDownloadSegments, shardPath, listener); - logger.trace("Downloaded segments from remote store {}", toDownloadSegments); + logger.debug("Downloaded segment files from remote store {}", toDownloadSegments); } finally { indexShard.store().decRef(); indexShard.remoteStore().decRef(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index c071b22ba4cba..ffc4ab86661db 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -208,7 +208,7 @@ public SegmentReplicationState getSegmentReplicationState(ShardId shardId) { * @param replicaShard replica shard on which checkpoint is received */ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedCheckpoint, final IndexShard replicaShard) { - logger.trace(() -> new ParameterizedMessage("Replica received new replication checkpoint from primary [{}]", receivedCheckpoint)); + logger.debug(() -> new ParameterizedMessage("Replica received new replication checkpoint from primary [{}]", receivedCheckpoint)); // if the shard is in any state if (replicaShard.state().equals(IndexShardState.CLOSED)) { // ignore if shard is closed @@ -224,7 +224,7 @@ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedChe SegmentReplicationTarget ongoingReplicationTarget = onGoingReplications.getOngoingReplicationTarget(replicaShard.shardId()); if (ongoingReplicationTarget != null) { if (ongoingReplicationTarget.getCheckpoint().getPrimaryTerm() < receivedCheckpoint.getPrimaryTerm()) { - logger.trace( + logger.debug( () -> new ParameterizedMessage( "Cancelling ongoing replication {} from old primary with primary term {}", ongoingReplicationTarget.description(), @@ -233,7 +233,7 @@ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedChe ); ongoingReplicationTarget.cancel("Cancelling stuck target after new primary"); } else { - logger.trace( + logger.debug( () -> new ParameterizedMessage( "Ignoring new replication checkpoint - shard is currently replicating to checkpoint {}", ongoingReplicationTarget.getCheckpoint() @@ -247,7 +247,7 @@ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedChe startReplication(replicaShard, receivedCheckpoint, new SegmentReplicationListener() { @Override public void onReplicationDone(SegmentReplicationState state) { - logger.trace( + logger.debug( () -> new ParameterizedMessage( "[shardId {}] [replication id {}] Replication complete to {}, timing data: {}", replicaShard.shardId().getId(), @@ -512,7 +512,7 @@ private void start(final long replicationId) { target.startReplication(new ActionListener<>() { @Override public void onResponse(Void o) { - logger.trace(() -> new ParameterizedMessage("Finished replicating {} marking as done.", target.description())); + logger.debug(() -> new ParameterizedMessage("Finished replicating {} marking as done.", target.description())); onGoingReplications.markAsDone(replicationId); if (target.state().getIndex().recoveredFileCount() != 0 && target.state().getIndex().recoveredBytes() != 0) { completedReplications.put(target.shardId(), target); @@ -521,6 +521,7 @@ public void onResponse(Void o) { @Override public void onFailure(Exception e) { + logger.debug("Replication failed {}", target.description()); if (e instanceof OpenSearchCorruptionException) { onGoingReplications.fail(replicationId, new ReplicationFailedException("Store corruption during replication", e), true); return; diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java index 76f0d333db977..11b01965c4af5 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java @@ -137,7 +137,7 @@ public String executor() { @Override public void handleResponse(ReplicationResponse response) { timer.stop(); - logger.trace( + logger.debug( () -> new ParameterizedMessage( "[shardId {}] Completed publishing checkpoint [{}], timing: {}", indexShard.shardId().getId(), @@ -152,7 +152,7 @@ public void handleResponse(ReplicationResponse response) { @Override public void handleException(TransportException e) { timer.stop(); - logger.trace("[shardId {}] Failed to publish checkpoint, timing: {}", indexShard.shardId().getId(), timer.time()); + logger.debug("[shardId {}] Failed to publish checkpoint, timing: {}", indexShard.shardId().getId(), timer.time()); task.setPhase("finished"); taskManager.unregister(task); if (ExceptionsHelper.unwrap( diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 220d468a03090..490ebda24bf60 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -100,6 +100,7 @@ import org.opensearch.core.compress.Compressor; import org.opensearch.core.compress.CompressorRegistry; import org.opensearch.core.compress.NotXContentException; +import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.index.snapshots.IndexShardSnapshotFailedException; import org.opensearch.core.util.BytesRefUtils; @@ -1125,7 +1126,11 @@ private void executeStaleShardDelete( new RemoteSegmentStoreDirectoryFactory( remoteStoreLockManagerFactory.getRepositoriesService(), threadPool - ).newDirectory(remoteStoreRepoForIndex, indexUUID, shardId).close(); + ).newDirectory( + remoteStoreRepoForIndex, + indexUUID, + new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.valueOf(shardId)) + ).close(); } } } @@ -1591,7 +1596,11 @@ private void executeOneStaleIndexDelete( new RemoteSegmentStoreDirectoryFactory( remoteStoreLockManagerFactory.getRepositoriesService(), threadPool - ).newDirectory(remoteStoreRepoForIndex, indexUUID, shardBlob.getKey()).close(); + ).newDirectory( + remoteStoreRepoForIndex, + indexUUID, + new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.valueOf(shardBlob.getKey())) + ).close(); } } } diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index c713ccdddd66a..415efd4ac23b6 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -152,7 +152,8 @@ public void testRemoteDirectoryInitThrowsException() throws IOException { mock(RemoteDirectory.class), remoteMetadataDirectory, mock(RemoteStoreLockManager.class), - mock(ThreadPool.class) + mock(ThreadPool.class), + shardId ); FilterDirectory remoteStoreFilterDirectory = new RemoteStoreRefreshListenerTests.TestFilterDirectory( new RemoteStoreRefreshListenerTests.TestFilterDirectory(remoteSegmentStoreDirectory) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index b016e8be67799..8d99d98fbaaf4 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -8,6 +8,8 @@ package org.opensearch.index.store; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -77,6 +79,7 @@ import static org.mockito.Mockito.when; public class RemoteSegmentStoreDirectoryTests extends IndexShardTestCase { + private static final Logger logger = LogManager.getLogger(RemoteSegmentStoreDirectoryTests.class); private RemoteDirectory remoteDataDirectory; private RemoteDirectory remoteMetadataDirectory; private RemoteStoreMetadataLockManager mdLockManager; @@ -97,13 +100,6 @@ public void setup() throws IOException { remoteMetadataDirectory = mock(RemoteDirectory.class); mdLockManager = mock(RemoteStoreMetadataLockManager.class); threadPool = mock(ThreadPool.class); - - remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( - remoteDataDirectory, - remoteMetadataDirectory, - mdLockManager, - threadPool - ); testUploadTracker = new TestUploadListener(); Settings indexSettings = Settings.builder() @@ -113,6 +109,13 @@ public void setup() throws IOException { ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService(); indexShard = newStartedShard(false, indexSettings, new NRTReplicationEngineFactory()); + remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDataDirectory, + remoteMetadataDirectory, + mdLockManager, + threadPool, + indexShard.shardId() + ); try (Store store = indexShard.store()) { segmentInfos = store.readLastCommittedSegmentsInfo(); } @@ -636,7 +639,8 @@ public void testCopyFilesFromMultipartIOException() throws Exception { remoteDataDirectory, remoteMetadataDirectory, mdLockManager, - threadPool + threadPool, + indexShard.shardId() ); populateMetadata(); diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 7a54c32248218..4505570b8ebe0 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -784,7 +784,7 @@ protected RemoteSegmentStoreDirectory createRemoteSegmentStoreDirectory(ShardId RemoteStoreLockManager remoteStoreLockManager = new RemoteStoreMetadataLockManager( new RemoteBufferedOutputDirectory(getBlobContainer(remoteShardPath.resolveIndex())) ); - return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, remoteStoreLockManager, threadPool); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, remoteStoreLockManager, threadPool, shardId); } private RemoteDirectory newRemoteDirectory(Path f) throws IOException { From 8807d7a801898c416dfefa532bb075a8edde0d5c Mon Sep 17 00:00:00 2001 From: Poojita Raj Date: Tue, 26 Sep 2023 15:04:14 -0700 Subject: [PATCH 074/388] Add workflow to add stalled label and message (#10197) * add workflow to add stalled label and message Signed-off-by: Poojita Raj --- .github/workflows/stalled.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/stalled.yml diff --git a/.github/workflows/stalled.yml b/.github/workflows/stalled.yml new file mode 100644 index 0000000000000..19ec9c9438bbe --- /dev/null +++ b/.github/workflows/stalled.yml @@ -0,0 +1,28 @@ +name: Label Stalled PRs +on: + schedule: + - cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT +permissions: + pull-requests: write +jobs: + stale: + if: github.repository == 'opensearch-project/OpenSearch' + runs-on: ubuntu-latest + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v2.1.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Stale PRs + uses: actions/stale@v8 + with: + repo-token: ${{ steps.github_app_token.outputs.token }} + stale-pr-label: 'stalled' + stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.' + days-before-pr-stale: 30 + days-before-issue-stale: -1 + days-before-pr-close: -1 + days-before-issue-close: -1 From 8f4f9951de817a58d18815c5dd35896dae2bced1 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 27 Sep 2023 16:41:37 -0400 Subject: [PATCH 075/388] [Feature] Support hashtag in token filter type_table rules parser (#10220) * Support hashtag in token filter type_table rules parser Signed-off-by: Louis Chu * Address comment from @reta Signed-off-by: Louis Chu --------- Signed-off-by: Louis Chu --- .../common/MappingCharFilterFactory.java | 2 +- ...rdDelimiterTokenFilterFactoryTestCase.java | 4 +- .../common/MappingCharFilterFactoryTests.java | 8 +- .../test/analysis-common/40_token_filters.yml | 126 ++++++++++++++++++ .../test/analysis-common/50_char_filters.yml | 1 + .../opensearch/index/analysis/Analysis.java | 27 +--- 6 files changed, 141 insertions(+), 27 deletions(-) diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/MappingCharFilterFactory.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/MappingCharFilterFactory.java index bd241de749f11..d6d9f8975f2fc 100644 --- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/MappingCharFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/MappingCharFilterFactory.java @@ -54,7 +54,7 @@ public class MappingCharFilterFactory extends AbstractCharFilterFactory implemen MappingCharFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name); - List> rules = Analysis.parseWordList(env, settings, "mappings", this::parse, false); + List> rules = Analysis.parseWordList(env, settings, "mappings", this::parse); if (rules == null) { throw new IllegalArgumentException("mapping requires either `mappings` or `mappings_path` to be configured"); } diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java index 94c7d63f2bee7..f37d5862b9d3f 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java @@ -211,8 +211,8 @@ private void createTokenFilterFactoryWithTypeTable(String[] rules) throws IOExce } public void testTypeTableParsingError() { - String[] rules = { "# This is a comment", "$ => DIGIT", "\\u200D => ALPHANUM", "abc => ALPHA" }; + String[] rules = { "# This is a comment", "# => ALPHANUM", "$ => DIGIT", "\\u200D => ALPHANUM", "abc => ALPHA" }; RuntimeException ex = expectThrows(RuntimeException.class, () -> createTokenFilterFactoryWithTypeTable(rules)); - assertEquals("Line [4]: Invalid mapping rule: [abc => ALPHA]. Only a single character is allowed.", ex.getMessage()); + assertEquals("Line [5]: Invalid mapping rule: [abc => ALPHA]. Only a single character is allowed.", ex.getMessage()); } } diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MappingCharFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MappingCharFilterFactoryTests.java index 387eb4a377007..28e041ac8c92d 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MappingCharFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/MappingCharFilterFactoryTests.java @@ -37,6 +37,7 @@ public static CharFilterFactory create(String... rules) throws IOException { public void testRulesOk() throws IOException { MappingCharFilterFactory mappingCharFilterFactory = (MappingCharFilterFactory) create( + "# This is a comment", "# => _hashtag_", ":) => _happy_", ":( => _sad_" @@ -64,7 +65,10 @@ public void testRuleError() { } public void testRulePartError() { - RuntimeException ex = expectThrows(RuntimeException.class, () -> create("# => _hashtag_", ":) => _happy_", "a:b")); - assertEquals("Line [3]: Invalid mapping rule : [a:b]", ex.getMessage()); + RuntimeException ex = expectThrows( + RuntimeException.class, + () -> create("# This is a comment", "# => _hashtag_", ":) => _happy_", "a:b") + ); + assertEquals("Line [4]: Invalid mapping rule : [a:b]", ex.getMessage()); } } diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml index e92cc0c4838c7..802c79c780689 100644 --- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml +++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml @@ -127,6 +127,69 @@ - match: { tokens.2.token: brown } - match: { tokens.3.token: fox } + - do: + indices.analyze: + body: + text: 'text1 #text2' + tokenizer: whitespace + filter: + - type: word_delimiter + split_on_numerics: false + type_table: + - "\\u0023 => ALPHANUM" + - length: { tokens: 2 } + - match: { tokens.0.token: text1 } + - match: { tokens.0.start_offset: 0 } + - match: { tokens.0.end_offset: 5 } + - match: { tokens.0.position: 0 } + - match: { tokens.1.token: "#text2" } + - match: { tokens.1.start_offset: 6 } + - match: { tokens.1.end_offset: 12 } + - match: { tokens.1.position: 1 } + + - do: + indices.analyze: + body: + text: 'text1 #text2' + tokenizer: whitespace + filter: + - type: word_delimiter + split_on_numerics: false + type_table: + - "# This is a comment" + - "# => ALPHANUM" + - length: { tokens: 2 } + - match: { tokens.0.token: text1 } + - match: { tokens.0.start_offset: 0 } + - match: { tokens.0.end_offset: 5 } + - match: { tokens.0.position: 0 } + - match: { tokens.1.token: "#text2" } + - match: { tokens.1.start_offset: 6 } + - match: { tokens.1.end_offset: 12 } + - match: { tokens.1.position: 1 } + + - do: + indices.analyze: + body: + text: 'text1 #text2' + tokenizer: whitespace + filter: + - type: word_delimiter + split_on_numerics: false + type_table: + - "# This is a comment" + - "# => ALPHANUM" + - "@ => ALPHANUM" + - length: { tokens: 2 } + - match: { tokens.0.token: text1 } + - match: { tokens.0.start_offset: 0 } + - match: { tokens.0.end_offset: 5 } + - match: { tokens.0.position: 0 } + - match: { tokens.1.token: "#text2" } + - match: { tokens.1.start_offset: 6 } + - match: { tokens.1.end_offset: 12 } + - match: { tokens.1.position: 1 } + --- "word_delimiter_graph": - do: @@ -231,6 +294,69 @@ - match: { detail.tokenfilters.0.tokens.5.end_offset: 19 } - match: { detail.tokenfilters.0.tokens.5.position: 5 } + - do: + indices.analyze: + body: + text: 'text1 #text2' + tokenizer: whitespace + filter: + - type: word_delimiter_graph + split_on_numerics: false + type_table: + - "\\u0023 => ALPHANUM" + - length: { tokens: 2 } + - match: { tokens.0.token: text1 } + - match: { tokens.0.start_offset: 0 } + - match: { tokens.0.end_offset: 5 } + - match: { tokens.0.position: 0 } + - match: { tokens.1.token: "#text2" } + - match: { tokens.1.start_offset: 6 } + - match: { tokens.1.end_offset: 12 } + - match: { tokens.1.position: 1 } + + - do: + indices.analyze: + body: + text: 'text1 #text2' + tokenizer: whitespace + filter: + - type: word_delimiter_graph + split_on_numerics: false + type_table: + - "# This is a comment" + - "# => ALPHANUM" + - length: { tokens: 2 } + - match: { tokens.0.token: text1 } + - match: { tokens.0.start_offset: 0 } + - match: { tokens.0.end_offset: 5 } + - match: { tokens.0.position: 0 } + - match: { tokens.1.token: "#text2" } + - match: { tokens.1.start_offset: 6 } + - match: { tokens.1.end_offset: 12 } + - match: { tokens.1.position: 1 } + + - do: + indices.analyze: + body: + text: 'text1 #text2' + tokenizer: whitespace + filter: + - type: word_delimiter_graph + split_on_numerics: false + type_table: + - "# This is a comment" + - "# => ALPHANUM" + - "@ => ALPHANUM" + - length: { tokens: 2 } + - match: { tokens.0.token: text1 } + - match: { tokens.0.start_offset: 0 } + - match: { tokens.0.end_offset: 5 } + - match: { tokens.0.position: 0 } + - match: { tokens.1.token: "#text2" } + - match: { tokens.1.start_offset: 6 } + - match: { tokens.1.end_offset: 12 } + - match: { tokens.1.position: 1 } + --- "unique": - do: diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml index 0078575ae8e57..5e266c10cba8f 100644 --- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml +++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml @@ -69,6 +69,7 @@ char_filter: - type: mapping mappings: + - "# This is a comment" - "# => _hashsign_" - "@ => _atsign_" - length: { tokens: 3 } diff --git a/server/src/main/java/org/opensearch/index/analysis/Analysis.java b/server/src/main/java/org/opensearch/index/analysis/Analysis.java index f4465c9dffac6..b9a219057f326 100644 --- a/server/src/main/java/org/opensearch/index/analysis/Analysis.java +++ b/server/src/main/java/org/opensearch/index/analysis/Analysis.java @@ -87,6 +87,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import static java.util.Collections.unmodifiableMap; @@ -98,6 +99,9 @@ public class Analysis { private static final Logger LOGGER = LogManager.getLogger(Analysis.class); + // Regular expression to support hashtag tokenization + private static final Pattern HASH_TAG_RULE_PATTERN = Pattern.compile("^\\s*#\\s*=>"); + public static CharArraySet parseStemExclusion(Settings settings, CharArraySet defaultStemExclusion) { String value = settings.get("stem_exclusion"); if ("_none_".equals(value)) { @@ -222,16 +226,6 @@ public static List parseWordList(Environment env, Settings settings, Stri return parseWordList(env, settings, settingPrefix + "_path", settingPrefix, parser); } - public static List parseWordList( - Environment env, - Settings settings, - String settingPrefix, - CustomMappingRuleParser parser, - boolean removeComments - ) { - return parseWordList(env, settings, settingPrefix + "_path", settingPrefix, parser, removeComments); - } - /** * Parses a list of words from the specified settings or from a file, with the given parser. * @@ -246,17 +240,6 @@ public static List parseWordList( String settingPath, String settingList, CustomMappingRuleParser parser - ) { - return parseWordList(env, settings, settingPath, settingList, parser, true); - } - - public static List parseWordList( - Environment env, - Settings settings, - String settingPath, - String settingList, - CustomMappingRuleParser parser, - boolean removeComments ) { List words = getWordList(env, settings, settingPath, settingList); if (words == null) { @@ -266,7 +249,7 @@ public static List parseWordList( int lineNum = 0; for (String word : words) { lineNum++; - if (removeComments == false || word.startsWith("#") == false) { + if (word.startsWith("#") == false || HASH_TAG_RULE_PATTERN.matcher(word).find() == true) { try { rules.add(parser.apply(word)); } catch (RuntimeException ex) { From 91f9c5dbf417e179c5ab00e49332c17c84661008 Mon Sep 17 00:00:00 2001 From: Ashish Date: Thu, 28 Sep 2023 08:35:04 +0530 Subject: [PATCH 076/388] Update recovery target access time during segment sync for remote index (#10252) * Update recovery target access time during segment sync for remote indexes Signed-off-by: Ashish Singh * Add Integration Test Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- .../opensearch/remotestore/RemoteStoreIT.java | 12 ++++++ .../opensearch/index/shard/IndexShard.java | 37 +++++++++++++------ .../opensearch/index/shard/StoreRecovery.java | 1 - .../recovery/PeerRecoveryTargetService.java | 2 +- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 0aa3a06a108f1..1fb5c2052aded 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexRequest; @@ -26,6 +27,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.translog.Translog.Durability; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -120,6 +122,16 @@ public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogFlush() throws Excep testPeerRecovery(randomIntBetween(2, 5), true); } + public void testPeerRecoveryWithLowActivityTimeout() throws Exception { + ClusterUpdateSettingsRequest req = new ClusterUpdateSettingsRequest().persistentSettings( + Settings.builder() + .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), "20kb") + .put(RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.getKey(), "1s") + ); + internalCluster().client().admin().cluster().updateSettings(req).get(); + testPeerRecovery(randomIntBetween(2, 5), true); + } + public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataRefresh() throws Exception { testPeerRecovery(1, false); } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index bbb7d3cf5f30f..d476e8b7c9288 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4726,11 +4726,21 @@ public void syncTranslogFilesFromRemoteTranslog() throws IOException { } /** - * Downloads segments from remote segment store. - * @param overrideLocal flag to override local segment files with those in remote store - * @throws IOException if exception occurs while reading segments from remote store + * Downloads segments from remote segment store + * @param overrideLocal flag to override local segment files with those in remote store. + * @throws IOException if exception occurs while reading segments from remote store. */ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal) throws IOException { + syncSegmentsFromRemoteSegmentStore(overrideLocal, () -> {}); + } + + /** + * Downloads segments from remote segment store along with updating the access time of the recovery target. + * @param overrideLocal flag to override local segment files with those in remote store. + * @param onFileSync runnable that updates the access time when run. + * @throws IOException if exception occurs while reading segments from remote store. + */ + public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runnable onFileSync) throws IOException { assert indexSettings.isRemoteStoreEnabled(); logger.trace("Downloading segments from remote segment store"); RemoteSegmentStoreDirectory remoteDirectory = getRemoteDirectory(); @@ -4761,7 +4771,7 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal) throws IOE } else { storeDirectory = store.directory(); } - copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal); + copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal, onFileSync); if (remoteSegmentMetadata != null) { final SegmentInfos infosSnapshot = store.buildSegmentInfos( @@ -4821,7 +4831,8 @@ public void syncSegmentsFromGivenRemoteSegmentStore( sourceRemoteDirectory, remoteDirectory, uploadedSegments, - overrideLocal + overrideLocal, + () -> {} ); if (segmentsNFile != null) { try ( @@ -4854,7 +4865,8 @@ private String copySegmentFiles( RemoteSegmentStoreDirectory sourceRemoteDirectory, RemoteSegmentStoreDirectory targetRemoteDirectory, Map uploadedSegments, - boolean overrideLocal + boolean overrideLocal, + final Runnable onFileSync ) throws IOException { Set toDownloadSegments = new HashSet<>(); Set skippedSegments = new HashSet<>(); @@ -4883,9 +4895,7 @@ private String copySegmentFiles( if (toDownloadSegments.isEmpty() == false) { try { - final PlainActionFuture completionListener = PlainActionFuture.newFuture(); - downloadSegments(storeDirectory, sourceRemoteDirectory, targetRemoteDirectory, toDownloadSegments, completionListener); - completionListener.actionGet(); + downloadSegments(storeDirectory, sourceRemoteDirectory, targetRemoteDirectory, toDownloadSegments, onFileSync); } catch (Exception e) { throw new IOException("Error occurred when downloading segments from remote store", e); } @@ -4903,22 +4913,25 @@ private void downloadSegments( RemoteSegmentStoreDirectory sourceRemoteDirectory, RemoteSegmentStoreDirectory targetRemoteDirectory, Set toDownloadSegments, - ActionListener completionListener + final Runnable onFileSync ) { - final Path indexPath = store.shardPath() == null ? null : store.shardPath().resolveIndex(); + final PlainActionFuture completionListener = PlainActionFuture.newFuture(); final GroupedActionListener batchDownloadListener = new GroupedActionListener<>( ActionListener.map(completionListener, v -> null), toDownloadSegments.size() ); final ActionListener segmentsDownloadListener = ActionListener.map(batchDownloadListener, fileName -> { + onFileSync.run(); if (targetRemoteDirectory != null) { targetRemoteDirectory.copyFrom(storeDirectory, fileName, fileName, IOContext.DEFAULT); } return null; }); - toDownloadSegments.forEach(file -> sourceRemoteDirectory.copyTo(file, storeDirectory, indexPath, segmentsDownloadListener)); + final Path indexPath = store.shardPath() == null ? null : store.shardPath().resolveIndex(); + toDownloadSegments.forEach(file -> { sourceRemoteDirectory.copyTo(file, storeDirectory, indexPath, segmentsDownloadListener); }); + completionListener.actionGet(); } private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) { diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index f429d94f7f96c..c0211e1257c8e 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -536,7 +536,6 @@ private void recoverFromRemoteStore(IndexShard indexShard) throws IndexShardReco try { // Download segments from remote segment store indexShard.syncSegmentsFromRemoteSegmentStore(true); - indexShard.syncTranslogFilesFromRemoteTranslog(); // On index creation, the only segment file that is created is segments_N. We can safely discard this file diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index 1ea32f4e355e3..65c3e976d17fe 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -245,7 +245,7 @@ private void doRecovery(final long recoveryId, final StartRecoveryRequest preExi indexShard.prepareForIndexRecovery(); final boolean hasRemoteSegmentStore = indexShard.indexSettings().isRemoteStoreEnabled(); if (hasRemoteSegmentStore) { - indexShard.syncSegmentsFromRemoteSegmentStore(false); + indexShard.syncSegmentsFromRemoteSegmentStore(false, recoveryTarget::setLastAccessTime); } final boolean hasRemoteTranslog = recoveryTarget.state().getPrimary() == false && indexShard.isRemoteTranslogEnabled(); final boolean hasNoTranslog = indexShard.indexSettings().isRemoteSnapshot(); From d656e3db592f29466d35452867caa241f5429485 Mon Sep 17 00:00:00 2001 From: Rohit Ashiwal Date: Thu, 28 Sep 2023 18:33:41 +0530 Subject: [PATCH 077/388] Indexing: add Doc Status Counter (#8716) Currently, Opensearch returns a 200 OK response code for a Bulk API call, even though there can be partial/complete failures within the request E2E. This provides doc level stats with respect to the rest status code as 2xx, 4xx, 5xx etc. Signed-off-by: Rohit Ashiwal --- CHANGELOG.md | 1 + .../org/opensearch/core/rest/RestStatus.java | 9 + .../test/nodes.stats/11_indices_metrics.yml | 29 ++ .../org/opensearch/nodestats/NodeStatsIT.java | 276 ++++++++++++++++++ .../action/bulk/TransportBulkAction.java | 47 ++- .../action/update/TransportUpdateAction.java | 25 +- .../opensearch/index/shard/IndexingStats.java | 123 +++++++- .../index/shard/InternalIndexingStats.java | 3 +- .../opensearch/indices/IndicesService.java | 10 + ...ActionIndicesThatCannotBeCreatedTests.java | 1 + .../bulk/TransportBulkActionIngestTests.java | 1 + .../action/bulk/TransportBulkActionTests.java | 3 + .../bulk/TransportBulkActionTookTests.java | 1 + .../org/opensearch/core/RestStatusTests.java | 91 ++++++ .../index/shard/IndexingStatsTests.java | 141 +++++++++ .../snapshots/SnapshotResiliencyTests.java | 1 + 16 files changed, 744 insertions(+), 18 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java create mode 100644 server/src/test/java/org/opensearch/core/RestStatusTests.java create mode 100644 server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 780db204aa1aa..b0b3ce261d517 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666)) - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131)) - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) +- Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java b/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java index 313bc23bedc90..8441ce8b1b622 100644 --- a/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java +++ b/libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java @@ -527,6 +527,15 @@ public int getStatus() { return status; } + /** + * Get category class of a rest status code. + * + * @return Integer representing class category of the concrete rest status code + */ + public int getStatusFamilyCode() { + return status / 100; + } + public static RestStatus readFrom(StreamInput in) throws IOException { return RestStatus.valueOf(in.readString()); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 1f1f42890355e..3f79227ce64e8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -138,6 +138,35 @@ - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery +--- +"Metric - indexing doc_status": + - skip: + version: " - 2.99.99" + reason: "To be introduced in future release :: TODO: change if/when we backport to 2.x" + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: { metric: indices, index_metric: indexing } + + - is_false: nodes.$node_id.indices.docs + - is_false: nodes.$node_id.indices.store + - is_true: nodes.$node_id.indices.indexing + - is_true: nodes.$node_id.indices.indexing.doc_status + - is_false: nodes.$node_id.indices.get + - is_false: nodes.$node_id.indices.search + - is_false: nodes.$node_id.indices.merges + - is_false: nodes.$node_id.indices.refresh + - is_false: nodes.$node_id.indices.flush + - is_false: nodes.$node_id.indices.warmer + - is_false: nodes.$node_id.indices.query_cache + - is_false: nodes.$node_id.indices.fielddata + - is_false: nodes.$node_id.indices.completion + - is_false: nodes.$node_id.indices.segments + - is_false: nodes.$node_id.indices.translog + - is_false: nodes.$node_id.indices.recovery --- "Metric - recovery": diff --git a/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java new file mode 100644 index 0000000000000..f270cb1399072 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java @@ -0,0 +1,276 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.nodestats; + +import org.opensearch.ExceptionsHelper; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.engine.DocumentMissingException; +import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.hamcrest.MatcherAssert; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +@ClusterScope(scope = Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) +public class NodeStatsIT extends OpenSearchIntegTestCase { + + private final DocStatusStats expectedDocStatusStats = new DocStatusStats(); + private static final String FIELD = "dummy_field"; + private static final String VALUE = "dummy_value"; + private static final Map SOURCE = singletonMap(FIELD, VALUE); + + public void testNodeIndicesStatsDocStatusStatsIndexBulk() { + { // Testing Index + final String INDEX = "test_index"; + final String ID = "id"; + { // Testing Normal Index + IndexResponse response = client().index(new IndexRequest(INDEX).id(ID).source(SOURCE)).actionGet(); + updateExpectedDocStatusCounter(response); + + MatcherAssert.assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertDocStatusStats(); + } + { // Testing Missing Alias + updateExpectedDocStatusCounter( + expectThrows( + IndexNotFoundException.class, + () -> client().index(new IndexRequest(INDEX).id("missing_alias").setRequireAlias(true).source(SOURCE)).actionGet() + ) + ); + assertDocStatusStats(); + } + { + // Test Missing Pipeline: Ingestion failure, not Indexing failure + expectThrows( + IllegalArgumentException.class, + () -> client().index(new IndexRequest(INDEX).id("missing_pipeline").setPipeline("missing").source(SOURCE)).actionGet() + ); + assertDocStatusStats(); + } + { // Testing Version Conflict + final String docId = "version_conflict"; + + updateExpectedDocStatusCounter(client().index(new IndexRequest(INDEX).id(docId).source(SOURCE)).actionGet()); + updateExpectedDocStatusCounter( + expectThrows( + VersionConflictEngineException.class, + () -> client().index(new IndexRequest(INDEX).id(docId).source(SOURCE).setIfSeqNo(1L).setIfPrimaryTerm(99L)) + .actionGet() + ) + ); + assertDocStatusStats(); + } + } + { // Testing Bulk + final String INDEX = "bulk_index"; + + int sizeOfIndexRequests = scaledRandomIntBetween(10, 20); + int sizeOfDeleteRequests = scaledRandomIntBetween(5, sizeOfIndexRequests); + int sizeOfNotFoundRequests = scaledRandomIntBetween(5, sizeOfIndexRequests); + + BulkRequest bulkRequest = new BulkRequest(); + + for (int i = 0; i < sizeOfIndexRequests; ++i) { + bulkRequest.add(new IndexRequest(INDEX).id(String.valueOf(i)).source(SOURCE)); + } + + BulkResponse response = client().bulk(bulkRequest).actionGet(); + + MatcherAssert.assertThat(response.hasFailures(), equalTo(false)); + MatcherAssert.assertThat(response.getItems().length, equalTo(sizeOfIndexRequests)); + + for (BulkItemResponse itemResponse : response.getItems()) { + updateExpectedDocStatusCounter(itemResponse.getResponse()); + } + + refresh(INDEX); + bulkRequest.requests().clear(); + + for (int i = 0; i < sizeOfDeleteRequests; ++i) { + bulkRequest.add(new DeleteRequest(INDEX, String.valueOf(i))); + } + for (int i = 0; i < sizeOfNotFoundRequests; ++i) { + bulkRequest.add(new DeleteRequest(INDEX, String.valueOf(25 + i))); + } + + response = client().bulk(bulkRequest).actionGet(); + + MatcherAssert.assertThat(response.hasFailures(), equalTo(false)); + MatcherAssert.assertThat(response.getItems().length, equalTo(sizeOfDeleteRequests + sizeOfNotFoundRequests)); + + for (BulkItemResponse itemResponse : response.getItems()) { + updateExpectedDocStatusCounter(itemResponse.getResponse()); + } + + refresh(INDEX); + assertDocStatusStats(); + } + } + + public void testNodeIndicesStatsDocStatusStatsCreateDeleteUpdate() { + { // Testing Create + final String INDEX = "create_index"; + final String ID = "id"; + { // Testing Creation + IndexResponse response = client().index(new IndexRequest(INDEX).id(ID).source(SOURCE).create(true)).actionGet(); + updateExpectedDocStatusCounter(response); + + MatcherAssert.assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertDocStatusStats(); + } + { // Testing Version Conflict + final String docId = "version_conflict"; + + updateExpectedDocStatusCounter(client().index(new IndexRequest(INDEX).id(docId).source(SOURCE)).actionGet()); + updateExpectedDocStatusCounter( + expectThrows( + VersionConflictEngineException.class, + () -> client().index(new IndexRequest(INDEX).id(docId).source(SOURCE).create(true)).actionGet() + ) + ); + assertDocStatusStats(); + } + } + { // Testing Delete + final String INDEX = "delete_index"; + final String ID = "id"; + { // Testing Deletion + IndexResponse response = client().index(new IndexRequest(INDEX).id(ID).source(SOURCE)).actionGet(); + updateExpectedDocStatusCounter(response); + + DeleteResponse deleteResponse = client().delete(new DeleteRequest(INDEX, ID)).actionGet(); + updateExpectedDocStatusCounter(deleteResponse); + + MatcherAssert.assertThat(response.getSeqNo(), greaterThanOrEqualTo(0L)); + MatcherAssert.assertThat(deleteResponse.getResult(), equalTo(DocWriteResponse.Result.DELETED)); + assertDocStatusStats(); + } + { // Testing Non-Existing Doc + updateExpectedDocStatusCounter(client().delete(new DeleteRequest(INDEX, "does_not_exist")).actionGet()); + assertDocStatusStats(); + } + { // Testing Version Conflict + final String docId = "version_conflict"; + + updateExpectedDocStatusCounter(client().index(new IndexRequest(INDEX).id(docId).source(SOURCE)).actionGet()); + updateExpectedDocStatusCounter( + expectThrows( + VersionConflictEngineException.class, + () -> client().delete(new DeleteRequest(INDEX, docId).setIfSeqNo(2L).setIfPrimaryTerm(99L)).actionGet() + ) + ); + + assertDocStatusStats(); + } + } + { // Testing Update + final String INDEX = "update_index"; + final String ID = "id"; + { // Testing Not Found + updateExpectedDocStatusCounter( + expectThrows( + DocumentMissingException.class, + () -> client().update(new UpdateRequest(INDEX, ID).doc(SOURCE)).actionGet() + ) + ); + assertDocStatusStats(); + } + { // Testing NoOp Update + updateExpectedDocStatusCounter(client().index(new IndexRequest(INDEX).id(ID).source(SOURCE)).actionGet()); + + UpdateResponse response = client().update(new UpdateRequest(INDEX, ID).doc(SOURCE)).actionGet(); + updateExpectedDocStatusCounter(response); + + MatcherAssert.assertThat(response.getResult(), equalTo(DocWriteResponse.Result.NOOP)); + assertDocStatusStats(); + } + { // Testing Update + final String UPDATED_VALUE = "updated_value"; + UpdateResponse response = client().update(new UpdateRequest(INDEX, ID).doc(singletonMap(FIELD, UPDATED_VALUE))).actionGet(); + updateExpectedDocStatusCounter(response); + + MatcherAssert.assertThat(response.getResult(), equalTo(DocWriteResponse.Result.UPDATED)); + assertDocStatusStats(); + } + { // Testing Missing Alias + updateExpectedDocStatusCounter( + expectThrows( + IndexNotFoundException.class, + () -> client().update(new UpdateRequest(INDEX, ID).setRequireAlias(true).doc(new IndexRequest().source(SOURCE))) + .actionGet() + ) + ); + assertDocStatusStats(); + } + { // Testing Version Conflict + final String docId = "version_conflict"; + + updateExpectedDocStatusCounter(client().index(new IndexRequest(INDEX).id(docId).source(SOURCE)).actionGet()); + updateExpectedDocStatusCounter( + expectThrows( + VersionConflictEngineException.class, + () -> client().update(new UpdateRequest(INDEX, docId).doc(SOURCE).setIfSeqNo(2L).setIfPrimaryTerm(99L)).actionGet() + ) + ); + assertDocStatusStats(); + } + } + } + + private void assertDocStatusStats() { + DocStatusStats docStatusStats = client().admin() + .cluster() + .prepareNodesStats() + .execute() + .actionGet() + .getNodes() + .get(0) + .getIndices() + .getIndexing() + .getTotal() + .getDocStatusStats(); + + assertTrue( + Arrays.equals( + docStatusStats.getDocStatusCounter(), + expectedDocStatusStats.getDocStatusCounter(), + Comparator.comparingLong(AtomicLong::longValue) + ) + ); + } + + private void updateExpectedDocStatusCounter(DocWriteResponse r) { + expectedDocStatusStats.inc(r.status()); + } + + private void updateExpectedDocStatusCounter(Exception e) { + expectedDocStatusStats.inc(ExceptionsHelper.status(e)); + } + +} diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java index d8b7facc09df2..726ba7ba119af 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java @@ -78,7 +78,9 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.VersionType; import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; import org.opensearch.indices.IndexClosedException; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; import org.opensearch.ingest.IngestService; import org.opensearch.node.NodeClosedException; @@ -129,6 +131,7 @@ public class TransportBulkAction extends HandledTransportAction> entry : requestsByShard.entrySet()) { final ShardId shardId = entry.getKey(); final List requests = entry.getValue(); @@ -632,8 +650,11 @@ public void onResponse(BulkShardResponse bulkShardResponse) { if (bulkItemResponse.getResponse() != null) { bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); } + + docStatusStats.inc(bulkItemResponse.status()); responses.set(bulkItemResponse.getItemId(), bulkItemResponse); } + if (counter.decrementAndGet() == 0) { finishHim(); } @@ -644,22 +665,24 @@ public void onFailure(Exception e) { // create failures for all relevant requests for (BulkItemRequest request : requests) { final String indexName = concreteIndices.getConcreteIndex(request.index()).getName(); - DocWriteRequest docWriteRequest = request.request(); - responses.set( + final DocWriteRequest docWriteRequest = request.request(); + final BulkItemResponse bulkItemResponse = new BulkItemResponse( request.id(), - new BulkItemResponse( - request.id(), - docWriteRequest.opType(), - new BulkItemResponse.Failure(indexName, docWriteRequest.id(), e) - ) + docWriteRequest.opType(), + new BulkItemResponse.Failure(indexName, docWriteRequest.id(), e) ); + + docStatusStats.inc(bulkItemResponse.status()); + responses.set(request.id(), bulkItemResponse); } + if (counter.decrementAndGet() == 0) { finishHim(); } } private void finishHim() { + indicesService.addDocStatusStats(docStatusStats); listener.onResponse( new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)) ); @@ -766,6 +789,10 @@ void executeBulk( final AtomicArray responses, Map indicesThatCannotBeCreated ) { + /* + * We are not wrapping the listener here to capture the response codes for performance benefits. It will + * be saving us an iteration over the responses array + */ new BulkOperation(task, bulkRequest, listener, responses, startTimeNanos, indicesThatCannotBeCreated).run(); } diff --git a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java index 95735f71a38e7..819112eb497f6 100644 --- a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java @@ -32,6 +32,7 @@ package org.opensearch.action.update; +import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionRunnable; import org.opensearch.action.DocWriteRequest; @@ -62,11 +63,13 @@ import org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; import org.opensearch.indices.IndicesService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -154,10 +157,13 @@ public static void resolveAndValidateRouting(Metadata metadata, String concreteI @Override protected void doExecute(Task task, final UpdateRequest request, final ActionListener listener) { if (request.isRequireAlias() && (clusterService.state().getMetadata().hasAlias(request.index()) == false)) { - throw new IndexNotFoundException( + IndexNotFoundException e = new IndexNotFoundException( "[" + DocWriteRequest.REQUIRE_ALIAS + "] request flag is [true] and [" + request.index() + "] is not an alias", request.index() ); + + incDocStatusStats(e); + throw e; } // if we don't have a master, we don't have metadata, that's fine, let it find a cluster-manager using create index API if (autoCreateIndex.shouldAutoCreate(request.index(), clusterService.state())) { @@ -193,7 +199,10 @@ public void onFailure(Exception e) { } private void innerExecute(final Task task, final UpdateRequest request, final ActionListener listener) { - super.doExecute(task, request, listener); + super.doExecute(task, request, ActionListener.wrap(listener::onResponse, e -> { + incDocStatusStats(e); + listener.onFailure(e); + })); } @Override @@ -330,7 +339,13 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< shard.noopUpdate(); } } + + DocStatusStats stats = new DocStatusStats(); + stats.inc(RestStatus.OK); + + indicesService.addDocStatusStats(stats); listener.onResponse(update); + break; default: throw new IllegalStateException("Illegal result " + result.getResponseResult()); @@ -361,4 +376,10 @@ private void handleUpdateFailureWithRetry( } listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause)); } + + private void incDocStatusStats(final Exception e) { + DocStatusStats stats = new DocStatusStats(); + stats.inc(ExceptionsHelper.status(e)); + indicesService.addDocStatusStats(stats); + } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java index 8953ef38da51b..f1abea81a6511 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java @@ -37,6 +37,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -44,6 +45,7 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; /** * Tracks indexing statistics @@ -59,6 +61,89 @@ public class IndexingStats implements Writeable, ToXContentFragment { */ public static class Stats implements Writeable, ToXContentFragment { + /** + * Tracks item level rest category class codes during indexing + * + * @opensearch.internal + */ + public static class DocStatusStats implements Writeable, ToXContentFragment { + + final AtomicLong[] docStatusCounter; + + public DocStatusStats() { + docStatusCounter = new AtomicLong[5]; + for (int i = 0; i < docStatusCounter.length; ++i) { + docStatusCounter[i] = new AtomicLong(0); + } + } + + public DocStatusStats(StreamInput in) throws IOException { + docStatusCounter = in.readArray(i -> new AtomicLong(i.readLong()), AtomicLong[]::new); + + assert docStatusCounter.length == 5 : "Length of incoming array should be 5! Got " + docStatusCounter.length; + } + + /** + * Increment counter for status + * + * @param status {@link RestStatus} + */ + public void inc(final RestStatus status) { + add(status, 1L); + } + + /** + * Increment counter for status by count + * + * @param status {@link RestStatus} + * @param delta The value to add + */ + void add(final RestStatus status, final long delta) { + docStatusCounter[status.getStatusFamilyCode() - 1].addAndGet(delta); + } + + /** + * Accumulate stats from the passed Object + * + * @param stats Instance storing {@link DocStatusStats} + */ + public void add(final DocStatusStats stats) { + if (null == stats) { + return; + } + + for (int i = 0; i < docStatusCounter.length; ++i) { + docStatusCounter[i].addAndGet(stats.docStatusCounter[i].longValue()); + } + } + + public AtomicLong[] getDocStatusCounter() { + return docStatusCounter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.DOC_STATUS); + + for (int i = 0; i < docStatusCounter.length; ++i) { + long value = docStatusCounter[i].longValue(); + + if (value > 0) { + String key = i + 1 + "xx"; + builder.field(key, value); + } + } + + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeArray((o, v) -> o.writeLong(v.longValue()), docStatusCounter); + } + + } + private long indexCount; private long indexTimeInMillis; private long indexCurrent; @@ -69,8 +154,11 @@ public static class Stats implements Writeable, ToXContentFragment { private long noopUpdateCount; private long throttleTimeInMillis; private boolean isThrottled; + private final DocStatusStats docStatusStats; - Stats() {} + Stats() { + docStatusStats = new DocStatusStats(); + } public Stats(StreamInput in) throws IOException { indexCount = in.readVLong(); @@ -83,6 +171,12 @@ public Stats(StreamInput in) throws IOException { noopUpdateCount = in.readVLong(); isThrottled = in.readBoolean(); throttleTimeInMillis = in.readLong(); + + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + docStatusStats = in.readOptionalWriteable(DocStatusStats::new); + } else { + docStatusStats = null; + } } public Stats( @@ -95,7 +189,8 @@ public Stats( long deleteCurrent, long noopUpdateCount, boolean isThrottled, - long throttleTimeInMillis + long throttleTimeInMillis, + DocStatusStats docStatusStats ) { this.indexCount = indexCount; this.indexTimeInMillis = indexTimeInMillis; @@ -107,6 +202,7 @@ public Stats( this.noopUpdateCount = noopUpdateCount; this.isThrottled = isThrottled; this.throttleTimeInMillis = throttleTimeInMillis; + this.docStatusStats = docStatusStats; } public void add(Stats stats) { @@ -121,8 +217,10 @@ public void add(Stats stats) { noopUpdateCount += stats.noopUpdateCount; throttleTimeInMillis += stats.throttleTimeInMillis; - if (isThrottled != stats.isThrottled) { - isThrottled = true; // When combining if one is throttled set result to throttled. + isThrottled |= stats.isThrottled; // When combining if one is throttled set result to throttled. + + if (getDocStatusStats() != null) { + getDocStatusStats().add(stats.getDocStatusStats()); } } @@ -193,6 +291,10 @@ public long getNoopUpdateCount() { return noopUpdateCount; } + public DocStatusStats getDocStatusStats() { + return docStatusStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(indexCount); @@ -206,6 +308,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isThrottled); out.writeLong(throttleTimeInMillis); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalWriteable(docStatusStats); + } } @Override @@ -223,8 +328,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.IS_THROTTLED, isThrottled); builder.humanReadableField(Fields.THROTTLED_TIME_IN_MILLIS, Fields.THROTTLED_TIME, getThrottleTime()); + + if (getDocStatusStats() != null) { + getDocStatusStats().toXContent(builder, params); + } + return builder; } + } private final Stats totalStats; @@ -279,7 +390,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par * * @opensearch.internal */ - static final class Fields { + private static final class Fields { static final String INDEXING = "indexing"; static final String INDEX_TOTAL = "index_total"; static final String INDEX_TIME = "index_time"; @@ -294,6 +405,7 @@ static final class Fields { static final String IS_THROTTLED = "is_throttled"; static final String THROTTLED_TIME_IN_MILLIS = "throttle_time_in_millis"; static final String THROTTLED_TIME = "throttle_time"; + static final String DOC_STATUS = "doc_status"; } @Override @@ -303,4 +415,5 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } + } diff --git a/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java b/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java index d7e15dd3e40f5..55b65bb4be6d8 100644 --- a/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java +++ b/server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java @@ -154,7 +154,8 @@ IndexingStats.Stats stats(boolean isThrottled, long currentThrottleMillis) { deleteCurrent.count(), noopUpdates.count(), isThrottled, - TimeUnit.MILLISECONDS.toMillis(currentThrottleMillis) + TimeUnit.MILLISECONDS.toMillis(currentThrottleMillis), + new IndexingStats.Stats.DocStatusStats() ); } } diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 6f4c51940ce87..a72142e65c5e8 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -133,6 +133,7 @@ import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.IndexingStats; +import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; import org.opensearch.index.store.remote.filecache.FileCacheCleaner; import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; @@ -1058,6 +1059,15 @@ public IndicesQueryCache getIndicesQueryCache() { return indicesQueryCache; } + /** + * Accumulate stats from the passed Object + * + * @param stats Instance storing {@link DocStatusStats} + */ + public void addDocStatusStats(final DocStatusStats stats) { + oldShardsStats.indexingStats.getTotal().getDocStatusStats().add(stats); + } + /** * Statistics for old shards * diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index 9ef0f85893fc8..0f67eff26cbde 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -154,6 +154,7 @@ private void indicesThatCannotBeCreatedTestCase( Settings.EMPTY, new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null) ), + null, new SystemIndices(emptyMap()) ) { @Override diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java index 5410d6a88a5b9..515f6eae28a34 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java @@ -171,6 +171,7 @@ class TestTransportBulkAction extends TransportBulkAction { SETTINGS, new ClusterService(SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null) ), + null, new SystemIndices(emptyMap()) ); } diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java index 098bd8e8d8cfe..10cad6fb147a2 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java @@ -60,6 +60,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.VersionType; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.SystemIndices; import org.opensearch.telemetry.tracing.noop.NoopTracer; @@ -88,6 +89,7 @@ import static org.opensearch.test.ClusterServiceUtils.createClusterService; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; public class TransportBulkActionTests extends OpenSearchTestCase { @@ -115,6 +117,7 @@ class TestTransportBulkAction extends TransportBulkAction { new Resolver(), new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(), new SystemIndices(emptyMap())), new IndexingPressureService(Settings.EMPTY, clusterService), + mock(IndicesService.class), new SystemIndices(emptyMap()) ); } diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java index 829eee45cac5b..852e3837e1e7a 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java @@ -280,6 +280,7 @@ static class TestTransportBulkAction extends TransportBulkAction { indexNameExpressionResolver, autoCreateIndex, new IndexingPressureService(Settings.EMPTY, clusterService), + null, new SystemIndices(emptyMap()), relativeTimeProvider ); diff --git a/server/src/test/java/org/opensearch/core/RestStatusTests.java b/server/src/test/java/org/opensearch/core/RestStatusTests.java new file mode 100644 index 0000000000000..f8dba99aa8b60 --- /dev/null +++ b/server/src/test/java/org/opensearch/core/RestStatusTests.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.core; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.action.ShardOperationFailedException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.PriorityQueue; + +public class RestStatusTests extends OpenSearchTestCase { + + public void testStatusReturns200ForNoFailures() { + int totalShards = randomIntBetween(1, 100); + int successfulShards = randomIntBetween(1, totalShards); + + assertEquals(RestStatus.OK, RestStatus.status(successfulShards, totalShards)); + } + + public void testStatusReturns503ForUnavailableShards() { + int totalShards = randomIntBetween(1, 100); + int successfulShards = 0; + + assertEquals(RestStatus.SERVICE_UNAVAILABLE, RestStatus.status(successfulShards, totalShards)); + } + + public void testStatusReturnsFailureStatusWhenFailuresExist() { + int totalShards = randomIntBetween(1, 100); + int successfulShards = 0; + + TestException[] failures = new TestException[totalShards]; + PriorityQueue heapOfFailures = new PriorityQueue<>((x, y) -> y.status().compareTo(x.status())); + + for (int i = 0; i < totalShards; ++i) { + /* + * Status here doesn't need to convey failure and is not as per rest + * contract. We're not testing the contract, but if status() returns + * the greatest rest code from the failures selection + */ + RestStatus status = randomFrom(RestStatus.values()); + TestException failure = new TestException(status); + + failures[i] = failure; + heapOfFailures.add(failure); + } + + assertEquals(heapOfFailures.peek().status(), RestStatus.status(successfulShards, totalShards, failures)); + } + + public void testSerialization() throws IOException { + final RestStatus status = randomFrom(RestStatus.values()); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + RestStatus.writeTo(out, status); + + try (StreamInput in = out.bytes().streamInput()) { + RestStatus deserializedStatus = RestStatus.readFrom(in); + + assertEquals(status, deserializedStatus); + } + } + } + + private static class TestException extends ShardOperationFailedException { + TestException(final RestStatus status) { + super("super-idx", randomInt(), "gone-fishing", status, new Throwable("cake")); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new IOException("not implemented"); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + throw new IOException("not implemented"); + } + } + +} diff --git a/server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java b/server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java new file mode 100644 index 0000000000000..acf482552c260 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/shard/IndexingStatsTests.java @@ -0,0 +1,141 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicLong; + +public class IndexingStatsTests extends OpenSearchTestCase { + + public void testSerialization() throws IOException { + IndexingStats stats = createTestInstance(); + + try (BytesStreamOutput out = new BytesStreamOutput()) { + stats.writeTo(out); + + try (StreamInput in = out.bytes().streamInput()) { + IndexingStats deserializedStats = new IndexingStats(in); + + if (stats.getTotal() == null) { + assertNull(deserializedStats.getTotal()); + return; + } + + IndexingStats.Stats totalStats = stats.getTotal(); + IndexingStats.Stats deserializedTotalStats = deserializedStats.getTotal(); + + assertEquals(totalStats.getIndexCount(), deserializedTotalStats.getIndexCount()); + assertEquals(totalStats.getIndexTime(), deserializedTotalStats.getIndexTime()); + assertEquals(totalStats.getIndexCurrent(), deserializedTotalStats.getIndexCurrent()); + assertEquals(totalStats.getIndexFailedCount(), deserializedTotalStats.getIndexFailedCount()); + assertEquals(totalStats.getDeleteCount(), deserializedTotalStats.getDeleteCount()); + assertEquals(totalStats.getDeleteTime(), deserializedTotalStats.getDeleteTime()); + assertEquals(totalStats.getDeleteCurrent(), deserializedTotalStats.getDeleteCurrent()); + assertEquals(totalStats.getNoopUpdateCount(), deserializedTotalStats.getNoopUpdateCount()); + assertEquals(totalStats.isThrottled(), deserializedTotalStats.isThrottled()); + assertEquals(totalStats.getThrottleTime(), deserializedTotalStats.getThrottleTime()); + + if (totalStats.getDocStatusStats() == null) { + assertNull(deserializedTotalStats.getDocStatusStats()); + return; + } + + IndexingStats.Stats.DocStatusStats docStatusStats = totalStats.getDocStatusStats(); + IndexingStats.Stats.DocStatusStats deserializedDocStatusStats = deserializedTotalStats.getDocStatusStats(); + + assertTrue( + Arrays.equals( + docStatusStats.getDocStatusCounter(), + deserializedDocStatusStats.getDocStatusCounter(), + Comparator.comparingLong(AtomicLong::longValue) + ) + ); + } + } + } + + public void testToXContentForIndexingStats() throws IOException { + IndexingStats stats = createTestInstance(); + IndexingStats.Stats totalStats = stats.getTotal(); + AtomicLong[] counter = totalStats.getDocStatusStats().getDocStatusCounter(); + + String expected = "{\"indexing\":{\"index_total\":" + + totalStats.getIndexCount() + + ",\"index_time_in_millis\":" + + totalStats.getIndexTime().getMillis() + + ",\"index_current\":" + + totalStats.getIndexCurrent() + + ",\"index_failed\":" + + totalStats.getIndexFailedCount() + + ",\"delete_total\":" + + totalStats.getDeleteCount() + + ",\"delete_time_in_millis\":" + + totalStats.getDeleteTime().getMillis() + + ",\"delete_current\":" + + totalStats.getDeleteCurrent() + + ",\"noop_update_total\":" + + totalStats.getNoopUpdateCount() + + ",\"is_throttled\":" + + totalStats.isThrottled() + + ",\"throttle_time_in_millis\":" + + totalStats.getThrottleTime().getMillis() + + ",\"doc_status\":{\"1xx\":" + + counter[0] + + ",\"2xx\":" + + counter[1] + + ",\"3xx\":" + + counter[2] + + ",\"4xx\":" + + counter[3] + + ",\"5xx\":" + + counter[4] + + "}}}"; + + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); + xContentBuilder.startObject(); + xContentBuilder = stats.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + xContentBuilder.endObject(); + + assertEquals(expected, xContentBuilder.toString()); + } + + private IndexingStats createTestInstance() { + IndexingStats.Stats.DocStatusStats docStatusStats = new IndexingStats.Stats.DocStatusStats(); + for (int i = 1; i < 6; ++i) { + docStatusStats.add(RestStatus.fromCode(i * 100), randomNonNegativeLong()); + } + + IndexingStats.Stats stats = new IndexingStats.Stats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomBoolean(), + randomNonNegativeLong(), + docStatusStats + ); + + return new IndexingStats(stats); + } + +} diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 2b432906ee128..09f5c1bea1a5e 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2210,6 +2210,7 @@ public void onFailure(final Exception e) { indexNameExpressionResolver, new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, new SystemIndices(emptyMap())), new IndexingPressureService(settings, clusterService), + mock(IndicesService.class), new SystemIndices(emptyMap()) ) ); From 7dc6683838951e1a211c6ff634d9eea93e2be3c2 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Thu, 28 Sep 2023 23:56:28 +0300 Subject: [PATCH 078/388] Fixed changelog for BC update (#10233) Signed-off-by: Andrey Pleskach --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0b3ce261d517..3d1ca935193c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208)) - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) - Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) From 61495bf0a915216f623d51807aeb3353caffd296 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 29 Sep 2023 13:40:02 -0700 Subject: [PATCH 079/388] Refactor async blob read to avoid blocking calls, support non multipart calls (#10192) Signed-off-by: Kunal Kotwani --- .../repositories/s3/S3BlobContainer.java | 89 +++++++----- .../s3/S3BlobStoreContainerTests.java | 129 +++++++++++++++--- 2 files changed, 163 insertions(+), 55 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index 2911a018df337..c6ae58371e15c 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -228,35 +228,50 @@ public void readBlobAsync(String blobName, ActionListener listener) try (AmazonAsyncS3Reference amazonS3Reference = SocketAccess.doPrivileged(blobStore::asyncClientReference)) { final S3AsyncClient s3AsyncClient = amazonS3Reference.get().client(); final String bucketName = blobStore.bucket(); + final String blobKey = buildKey(blobName); - final GetObjectAttributesResponse blobMetadata = getBlobMetadata(s3AsyncClient, bucketName, blobName).get(); + final CompletableFuture blobMetadataFuture = getBlobMetadata(s3AsyncClient, bucketName, blobKey); - final long blobSize = blobMetadata.objectSize(); - final int numberOfParts = blobMetadata.objectParts().totalPartsCount(); - final String blobChecksum = blobMetadata.checksum().checksumCRC32(); - - final List blobPartStreams = new ArrayList<>(); - final List> blobPartInputStreamFutures = new ArrayList<>(); - // S3 multipart files use 1 to n indexing - for (int partNumber = 1; partNumber <= numberOfParts; partNumber++) { - blobPartInputStreamFutures.add(getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobName, partNumber)); - } - - CompletableFuture.allOf(blobPartInputStreamFutures.toArray(CompletableFuture[]::new)).whenComplete((unused, throwable) -> { - if (throwable == null) { - listener.onResponse( - new ReadContext( - blobSize, - blobPartInputStreamFutures.stream().map(CompletableFuture::join).collect(Collectors.toList()), - blobChecksum - ) - ); - } else { + blobMetadataFuture.whenComplete((blobMetadata, throwable) -> { + if (throwable != null) { Exception ex = throwable.getCause() instanceof Exception ? (Exception) throwable.getCause() : new Exception(throwable.getCause()); listener.onFailure(ex); + return; + } + + final List> blobPartInputStreamFutures = new ArrayList<>(); + final long blobSize = blobMetadata.objectSize(); + final Integer numberOfParts = blobMetadata.objectParts() == null ? null : blobMetadata.objectParts().totalPartsCount(); + final String blobChecksum = blobMetadata.checksum().checksumCRC32(); + + if (numberOfParts == null) { + blobPartInputStreamFutures.add(getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, null)); + } else { + // S3 multipart files use 1 to n indexing + for (int partNumber = 1; partNumber <= numberOfParts; partNumber++) { + blobPartInputStreamFutures.add(getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, partNumber)); + } } + + CompletableFuture.allOf(blobPartInputStreamFutures.toArray(CompletableFuture[]::new)) + .whenComplete((unused, partThrowable) -> { + if (partThrowable == null) { + listener.onResponse( + new ReadContext( + blobSize, + blobPartInputStreamFutures.stream().map(CompletableFuture::join).collect(Collectors.toList()), + blobChecksum + ) + ); + } else { + Exception ex = partThrowable.getCause() instanceof Exception + ? (Exception) partThrowable.getCause() + : new Exception(partThrowable.getCause()); + listener.onFailure(ex); + } + }); }); } catch (Exception ex) { listener.onFailure(SdkException.create("Error occurred while fetching blob parts from the repository", ex)); @@ -685,41 +700,47 @@ static Tuple numberOfMultiparts(final long totalSize, final long par * the stream and its related metadata. * @param s3AsyncClient Async client to be utilized to fetch the object part * @param bucketName Name of the S3 bucket - * @param blobName Identifier of the blob for which the parts will be fetched - * @param partNumber Part number for the blob to be retrieved + * @param blobKey Identifier of the blob for which the parts will be fetched + * @param partNumber Optional part number for the blob to be retrieved * @return A future of {@link InputStreamContainer} containing the stream and stream metadata. */ CompletableFuture getBlobPartInputStreamContainer( S3AsyncClient s3AsyncClient, String bucketName, - String blobName, - int partNumber + String blobKey, + @Nullable Integer partNumber ) { - final GetObjectRequest.Builder getObjectRequestBuilder = GetObjectRequest.builder() - .bucket(bucketName) - .key(blobName) - .partNumber(partNumber); + final boolean isMultipartObject = partNumber != null; + final GetObjectRequest.Builder getObjectRequestBuilder = GetObjectRequest.builder().bucket(bucketName).key(blobKey); + + if (isMultipartObject) { + getObjectRequestBuilder.partNumber(partNumber); + } return SocketAccess.doPrivileged( () -> s3AsyncClient.getObject(getObjectRequestBuilder.build(), AsyncResponseTransformer.toBlockingInputStream()) - .thenApply(S3BlobContainer::transformResponseToInputStreamContainer) + .thenApply(response -> transformResponseToInputStreamContainer(response, isMultipartObject)) ); } /** * Transforms the stream response object from S3 into an {@link InputStreamContainer} * @param streamResponse Response stream object from S3 + * @param isMultipartObject Flag to denote a multipart object response * @return {@link InputStreamContainer} containing the stream and stream metadata */ // Package-Private for testing. - static InputStreamContainer transformResponseToInputStreamContainer(ResponseInputStream streamResponse) { + static InputStreamContainer transformResponseToInputStreamContainer( + ResponseInputStream streamResponse, + boolean isMultipartObject + ) { final GetObjectResponse getObjectResponse = streamResponse.response(); final String contentRange = getObjectResponse.contentRange(); final Long contentLength = getObjectResponse.contentLength(); - if (contentRange == null || contentLength == null) { + if ((isMultipartObject && contentRange == null) || contentLength == null) { throw SdkException.builder().message("Failed to fetch required metadata for blob part").build(); } - final Long offset = HttpRangeUtils.getStartOffsetFromRangeHeader(getObjectResponse.contentRange()); + final long offset = isMultipartObject ? HttpRangeUtils.getStartOffsetFromRangeHeader(getObjectResponse.contentRange()) : 0L; return new InputStreamContainer(streamResponse, getObjectResponse.contentLength(), offset); } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index a87c060dcc60a..9817d7cd520ef 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -81,7 +81,6 @@ import org.opensearch.common.io.InputStreamContainer; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeUnit; -import org.opensearch.repositories.s3.async.AsyncTransferManager; import org.opensearch.test.OpenSearchTestCase; import java.io.ByteArrayInputStream; @@ -100,7 +99,6 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -919,7 +917,7 @@ public void testListBlobsByPrefixInLexicographicOrderWithLimitGreaterThanNumberO testListBlobsByPrefixInLexicographicOrder(12, 2, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); } - public void testReadBlobAsync() throws Exception { + public void testReadBlobAsyncMultiPart() throws Exception { final String bucketName = randomAlphaOfLengthBetween(1, 10); final String blobName = randomAlphaOfLengthBetween(1, 10); final String checksum = randomAlphaOfLength(10); @@ -932,11 +930,7 @@ public void testReadBlobAsync() throws Exception { final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference( AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null) ); - final AsyncTransferManager asyncTransferManager = new AsyncTransferManager( - 10000L, - mock(ExecutorService.class), - mock(ExecutorService.class) - ); + final S3BlobStore blobStore = mock(S3BlobStore.class); final BlobPath blobPath = new BlobPath(); @@ -944,7 +938,6 @@ public void testReadBlobAsync() throws Exception { when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); when(blobStore.serverSideEncryption()).thenReturn(false); when(blobStore.asyncClientReference()).thenReturn(amazonAsyncS3Reference); - when(blobStore.getAsyncTransferManager()).thenReturn(asyncTransferManager); CompletableFuture getObjectAttributesResponseCompletableFuture = new CompletableFuture<>(); getObjectAttributesResponseCompletableFuture.complete( @@ -984,6 +977,60 @@ public void testReadBlobAsync() throws Exception { } } + public void testReadBlobAsyncSinglePart() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final String blobName = randomAlphaOfLengthBetween(1, 10); + final String checksum = randomAlphaOfLength(10); + + final int objectSize = 100; + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference( + AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null) + ); + final S3BlobStore blobStore = mock(S3BlobStore.class); + final BlobPath blobPath = new BlobPath(); + + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.serverSideEncryption()).thenReturn(false); + when(blobStore.asyncClientReference()).thenReturn(amazonAsyncS3Reference); + + CompletableFuture getObjectAttributesResponseCompletableFuture = new CompletableFuture<>(); + getObjectAttributesResponseCompletableFuture.complete( + GetObjectAttributesResponse.builder() + .checksum(Checksum.builder().checksumCRC32(checksum).build()) + .objectSize((long) objectSize) + .build() + ); + when(s3AsyncClient.getObjectAttributes(any(GetObjectAttributesRequest.class))).thenReturn( + getObjectAttributesResponseCompletableFuture + ); + + mockObjectResponse(s3AsyncClient, bucketName, blobName, objectSize); + + CountDownLatch countDownLatch = new CountDownLatch(1); + CountingCompletionListener readContextActionListener = new CountingCompletionListener<>(); + LatchedActionListener listener = new LatchedActionListener<>(readContextActionListener, countDownLatch); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + blobContainer.readBlobAsync(blobName, listener); + countDownLatch.await(); + + assertEquals(1, readContextActionListener.getResponseCount()); + assertEquals(0, readContextActionListener.getFailureCount()); + ReadContext readContext = readContextActionListener.getResponse(); + assertEquals(1, readContext.getNumberOfParts()); + assertEquals(checksum, readContext.getBlobChecksum()); + assertEquals(objectSize, readContext.getBlobSize()); + + InputStreamContainer inputStreamContainer = readContext.getPartStreams().stream().findFirst().get(); + assertEquals(objectSize, inputStreamContainer.getContentLength()); + assertEquals(0, inputStreamContainer.getOffset()); + assertEquals(objectSize, inputStreamContainer.getInputStream().readAllBytes().length); + + } + public void testReadBlobAsyncFailure() throws Exception { final String bucketName = randomAlphaOfLengthBetween(1, 10); final String blobName = randomAlphaOfLengthBetween(1, 10); @@ -996,11 +1043,7 @@ public void testReadBlobAsyncFailure() throws Exception { final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference( AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null) ); - final AsyncTransferManager asyncTransferManager = new AsyncTransferManager( - 10000L, - mock(ExecutorService.class), - mock(ExecutorService.class) - ); + final S3BlobStore blobStore = mock(S3BlobStore.class); final BlobPath blobPath = new BlobPath(); @@ -1008,7 +1051,6 @@ public void testReadBlobAsyncFailure() throws Exception { when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); when(blobStore.serverSideEncryption()).thenReturn(false); when(blobStore.asyncClientReference()).thenReturn(amazonAsyncS3Reference); - when(blobStore.getAsyncTransferManager()).thenReturn(asyncTransferManager); CompletableFuture getObjectAttributesResponseCompletableFuture = new CompletableFuture<>(); getObjectAttributesResponseCompletableFuture.complete( @@ -1071,7 +1113,7 @@ public void testGetBlobPartInputStream() throws Exception { final String blobName = randomAlphaOfLengthBetween(1, 10); final String bucketName = randomAlphaOfLengthBetween(1, 10); final long contentLength = 10L; - final String contentRange = "bytes 0-10/100"; + final String contentRange = "bytes 10-20/100"; final InputStream inputStream = ResponseInputStream.nullInputStream(); final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); @@ -1095,9 +1137,17 @@ public void testGetBlobPartInputStream() throws Exception { ) ).thenReturn(getObjectPartResponse); + // Header based offset in case of a multi part object request InputStreamContainer inputStreamContainer = blobContainer.getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobName, 0) .get(); + assertEquals(10, inputStreamContainer.getOffset()); + assertEquals(contentLength, inputStreamContainer.getContentLength()); + assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); + + // 0 offset in case of a single part object request + inputStreamContainer = blobContainer.getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobName, null).get(); + assertEquals(0, inputStreamContainer.getOffset()); assertEquals(contentLength, inputStreamContainer.getContentLength()); assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); @@ -1108,28 +1158,65 @@ public void testTransformResponseToInputStreamContainer() throws Exception { final long contentLength = 10L; final InputStream inputStream = ResponseInputStream.nullInputStream(); - final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); - GetObjectResponse getObjectResponse = GetObjectResponse.builder().contentLength(contentLength).build(); + // Exception when content range absent for multipart object ResponseInputStream responseInputStreamNoRange = new ResponseInputStream<>(getObjectResponse, inputStream); - assertThrows(SdkException.class, () -> S3BlobContainer.transformResponseToInputStreamContainer(responseInputStreamNoRange)); + assertThrows(SdkException.class, () -> S3BlobContainer.transformResponseToInputStreamContainer(responseInputStreamNoRange, true)); + + // No exception when content range absent for single part object + ResponseInputStream responseInputStreamNoRangeSinglePart = new ResponseInputStream<>( + getObjectResponse, + inputStream + ); + InputStreamContainer inputStreamContainer = S3BlobContainer.transformResponseToInputStreamContainer( + responseInputStreamNoRangeSinglePart, + false + ); + assertEquals(contentLength, inputStreamContainer.getContentLength()); + assertEquals(0, inputStreamContainer.getOffset()); + // Exception when length is absent getObjectResponse = GetObjectResponse.builder().contentRange(contentRange).build(); ResponseInputStream responseInputStreamNoContentLength = new ResponseInputStream<>( getObjectResponse, inputStream ); - assertThrows(SdkException.class, () -> S3BlobContainer.transformResponseToInputStreamContainer(responseInputStreamNoContentLength)); + assertThrows( + SdkException.class, + () -> S3BlobContainer.transformResponseToInputStreamContainer(responseInputStreamNoContentLength, true) + ); + // No exception when range and length both are present getObjectResponse = GetObjectResponse.builder().contentRange(contentRange).contentLength(contentLength).build(); ResponseInputStream responseInputStream = new ResponseInputStream<>(getObjectResponse, inputStream); - InputStreamContainer inputStreamContainer = S3BlobContainer.transformResponseToInputStreamContainer(responseInputStream); + inputStreamContainer = S3BlobContainer.transformResponseToInputStreamContainer(responseInputStream, true); assertEquals(contentLength, inputStreamContainer.getContentLength()); assertEquals(0, inputStreamContainer.getOffset()); assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); } + private void mockObjectResponse(S3AsyncClient s3AsyncClient, String bucketName, String blobName, int objectSize) { + + final InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(objectSize)); + + GetObjectResponse getObjectResponse = GetObjectResponse.builder().contentLength((long) objectSize).build(); + + CompletableFuture> getObjectPartResponse = new CompletableFuture<>(); + ResponseInputStream responseInputStream = new ResponseInputStream<>(getObjectResponse, inputStream); + getObjectPartResponse.complete(responseInputStream); + + GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(blobName).build(); + + when( + s3AsyncClient.getObject( + eq(getObjectRequest), + ArgumentMatchers.>>any() + ) + ).thenReturn(getObjectPartResponse); + + } + private void mockObjectPartResponse( S3AsyncClient s3AsyncClient, String bucketName, From be247669c4d5a4494958a508c2fcde4e36c48499 Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:21:37 -0700 Subject: [PATCH 080/388] [Search latency - Coordinator] Changing version check to 2.11 (#10280) * [Search latency - Coordinator] Changing version check to 2.11 Signed-off-by: Sagar Upadhyaya * [Port main] update version check as per v2.11.0 [Backport 2.x] Indexing: add Doc status counter (#10267) * Indexing: add Doc Status Counter (#8716) Currently, Opensearch returns a 200 OK response code for a Bulk API call, even though there can be partial/complete failures within the request E2E. This provides doc level stats with respect to the rest status code as 2xx, 4xx, 5xx etc. Signed-off-by: Rohit Ashiwal (cherry picked from commit d656e3db592f29466d35452867caa241f5429485) Signed-off-by: github-actions[bot] (cherry picked from commit 94173e3f8b343ba29db24e5682d1706bbda9f9a4) Signed-off-by: Rohit Ashiwal Signed-off-by: Sagar Upadhyaya --------- Signed-off-by: Sagar Upadhyaya Signed-off-by: Rohit Ashiwal Signed-off-by: github-actions[bot] --- .../rest-api-spec/test/nodes.stats/11_indices_metrics.yml | 4 ++-- .../java/org/opensearch/index/search/stats/SearchStats.java | 4 ++-- .../main/java/org/opensearch/index/shard/IndexingStats.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 3f79227ce64e8..784c7b52b18b4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -141,8 +141,8 @@ --- "Metric - indexing doc_status": - skip: - version: " - 2.99.99" - reason: "To be introduced in future release :: TODO: change if/when we backport to 2.x" + version: " - 2.10.99" + reason: "Doc Status Stats were introduced in v2.11.0" - do: nodes.info: {} - set: diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java index 14aaf7e58a59c..1f9144b28f286 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java @@ -242,7 +242,7 @@ private Stats(StreamInput in) throws IOException { pitCurrent = in.readVLong(); } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_11_0)) { this.requestStatsLongHolder = new RequestStatsLongHolder(); requestStatsLongHolder.requestStatsHolder = in.readMap(StreamInput::readString, PhaseStatsLongHolder::new); } @@ -437,7 +437,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(pitCurrent); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_11_0)) { if (requestStatsLongHolder == null) { requestStatsLongHolder = new RequestStatsLongHolder(); } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java index f1abea81a6511..89cbc59403faf 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java @@ -172,7 +172,7 @@ public Stats(StreamInput in) throws IOException { isThrottled = in.readBoolean(); throttleTimeInMillis = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_11_0)) { docStatusStats = in.readOptionalWriteable(DocStatusStats::new); } else { docStatusStats = null; @@ -308,7 +308,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isThrottled); out.writeLong(throttleTimeInMillis); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_11_0)) { out.writeOptionalWriteable(docStatusStats); } } From e1565821a28109c5c7f773e77764fc5104de7f88 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Sat, 30 Sep 2023 09:02:49 -0700 Subject: [PATCH 081/388] Pass parent filter to inner query in nested query (#10246) Pass parent filter to inner query so that inner query can utilize the information Signed-off-by: Heemin Kim --- CHANGELOG.md | 1 + .../index/query/NestedQueryBuilder.java | 3 +++ .../index/query/QueryShardContext.java | 9 +++++++++ .../index/query/NestedQueryBuilderTests.java | 20 +++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d1ca935193c3..f6c29d9b64f68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036)) - Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042)) - [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122)) +- Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java index 33fe96597b4f8..ac4fde7f06b16 100644 --- a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java @@ -318,10 +318,13 @@ protected Query doToQuery(QueryShardContext context) throws IOException { parentFilter = context.bitsetFilter(objectMapper.nestedTypeFilter()); } + BitSetProducer previousParentFilter = context.getParentFilter(); try { + context.setParentFilter(parentFilter); context.nestedScope().nextLevel(nestedObjectMapper); innerQuery = this.query.toQuery(context); } finally { + context.setParentFilter(previousParentFilter); context.nestedScope().previousLevel(); } diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java index 0bf05e633bba3..701484fbc8dc3 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java @@ -115,6 +115,7 @@ public class QueryShardContext extends QueryRewriteContext { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private final ValuesSourceRegistry valuesSourceRegistry; + private BitSetProducer parentFilter; public QueryShardContext( int shardId, @@ -622,4 +623,12 @@ public BitsetFilterCache getBitsetFilterCache() { public AggregationUsageService getUsageService() { return valuesSourceRegistry.getUsageService(); } + + public BitSetProducer getParentFilter() { + return parentFilter; + } + + public void setParentFilter(BitSetProducer parentFilter) { + this.parentFilter = parentFilter; + } } diff --git a/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java index 0d66654a70f08..62337264bc0b1 100644 --- a/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java @@ -66,6 +66,8 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.notNullValue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class NestedQueryBuilderTests extends AbstractQueryTestCase { @@ -411,4 +413,22 @@ public void testDisallowExpensiveQueries() { OpenSearchException e = expectThrows(OpenSearchException.class, () -> queryBuilder.toQuery(queryShardContext)); assertEquals("[joining] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", e.getMessage()); } + + public void testSetParentFilterInContext() throws Exception { + QueryShardContext queryShardContext = createShardContext(); + QueryBuilder innerQueryBuilder = spy(new MatchAllQueryBuilderTests().createTestQueryBuilder()); + when(innerQueryBuilder.toQuery(queryShardContext)).thenAnswer(invoke -> { + QueryShardContext context = invoke.getArgument(0); + if (context.getParentFilter() == null) { + throw new Exception("Expect parent filter to be non-null"); + } + return invoke.callRealMethod(); + }); + NestedQueryBuilder nqb = new NestedQueryBuilder("nested1", innerQueryBuilder, RandomPicks.randomFrom(random(), ScoreMode.values())); + + assertNull(queryShardContext.getParentFilter()); + nqb.rewrite(queryShardContext).toQuery(queryShardContext); + assertNull(queryShardContext.getParentFilter()); + verify(innerQueryBuilder).toQuery(queryShardContext); + } } From 797def6e50389d88634b6ca614ceaba407ee983f Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Sat, 30 Sep 2023 18:45:30 -0700 Subject: [PATCH 082/388] Upgrade Lucene to 9.8.0 (#10276) 9.8.0 was officially released this morning: https://lucene.apache.org/core/9_8_0/changes/Changes.html Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - libs/core/licenses/lucene-core-9.8.0.jar.sha1 | 1 + .../licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 | 1 + .../lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 | 1 + .../lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 | 1 + .../lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 | 1 + .../lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 | 1 + .../lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../licenses/lucene-analysis-stempel-9.8.0.jar.sha1 | 1 + .../lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - .../licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 | 1 + .../lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-analysis-common-9.8.0.jar.sha1 | 1 + .../lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 | 1 + server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-core-9.8.0.jar.sha1 | 1 + server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-grouping-9.8.0.jar.sha1 | 1 + .../licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-highlighter-9.8.0.jar.sha1 | 1 + server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-join-9.8.0.jar.sha1 | 1 + server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-memory-9.8.0.jar.sha1 | 1 + server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-misc-9.8.0.jar.sha1 | 1 + server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-queries-9.8.0.jar.sha1 | 1 + .../licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-queryparser-9.8.0.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.8.0.jar.sha1 | 1 + .../lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 | 1 + .../licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-spatial3d-9.8.0.jar.sha1 | 1 + server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 | 1 - server/licenses/lucene-suggest-9.8.0.jar.sha1 | 1 + 48 files changed, 25 insertions(+), 24 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.8.0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-core-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-join-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.8.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.8.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index f6c29d9b64f68..772198d5d0544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` - Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` - Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 71b3e267700b1..dae68940a7b7f 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.8.0-snapshot-95cdd2e +lucene = 9.8.0 bundled_jdk_vendor = adoptium bundled_jdk = 20.0.2+9 diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 70baf1270cd5d..0000000000000 --- a/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d2f7fbc5b2c49ca777a169d579f41082a9a57cc7 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.8.0.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..f9a3e2f3cbee6 --- /dev/null +++ b/libs/core/licenses/lucene-core-9.8.0.jar.sha1 @@ -0,0 +1 @@ +5e8421c5f8573bcf22e9265fc7e19469545a775a \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index c1daa91dd5433..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57e2b0cca55da8ad856dfd60be42e6daabbc98c3 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..892865a017f48 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 @@ -0,0 +1 @@ +7725476acfcb9bdfeff1b813ce15c39c6b857dc2 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 035b47c5f388c..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0deb3b85eadf831be17b48acab0785fd9d34fc44 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..ef410899981ca --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 @@ -0,0 +1 @@ +7133d34e92770f59eb28686f4d511b9f3f32e970 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 6ff5a433f0a4e..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9a204267d68ce4ba36bfddc366cd6865cf5e1378 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..46b83c9e40b3a --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 @@ -0,0 +1 @@ +be44282e1f6b91a0650fcceb558053d6bdd4863d \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index a65ab33a31e2a..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71e8e811f873ba2b47c7ecf9d890cbeac5b6be41 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..36664695a7818 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 @@ -0,0 +1 @@ +bd1f80d33346f7e588685484ef29a304db5190e4 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 04ab7b7e7adb8..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e1274273895365bd83391cc4b79f5264479f5de \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..003ccdf8b0727 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 @@ -0,0 +1 @@ +b9ffdc7a52d2087ecb03318ec06305b480cdfe82 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index cef3f97d03c51..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e634c8685edad2bdb5c13748b18c0c1a46bb63a3 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..e22eaa474016f --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 @@ -0,0 +1 @@ +f73e2007b133fb699e517ef13b4952844f0150d8 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 3e2dd19a9dd85..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0afdf2afacbae39414ed06325fbb4bed17c07a7d \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..1ebe42a2a2f56 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 @@ -0,0 +1 @@ +2c09cbc021a8f81a01600a1d2a999361e70f7aed \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 8c0544acd1ca0..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -166e2ea297182f7bf7070af02aacea9e6a3a19c8 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..3c4523d45c0f5 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 @@ -0,0 +1 @@ +b054f2c7b11fc7c5601b4c3cdf18aa7508612898 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 4ac89f2e792d7..0000000000000 --- a/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8c82be3d997d781bb72d6d0eadade064dd2cd6db \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.8.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..6ad304fa52c12 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.8.0.jar.sha1 @@ -0,0 +1 @@ +36f0363325ca7bf62c180160d1ed5165c7c37795 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 624b5174a444f..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4c261d17c681c0d91171c67e192abfef59adea2e \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..f104c4207d390 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 @@ -0,0 +1 @@ +e98fb408028f40170e6d87c16422bfdc0bb2e392 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 70baf1270cd5d..0000000000000 --- a/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d2f7fbc5b2c49ca777a169d579f41082a9a57cc7 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.8.0.jar.sha1 b/server/licenses/lucene-core-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..f9a3e2f3cbee6 --- /dev/null +++ b/server/licenses/lucene-core-9.8.0.jar.sha1 @@ -0,0 +1 @@ +5e8421c5f8573bcf22e9265fc7e19469545a775a \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 20ddb9ae3ef27..0000000000000 --- a/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d1cf3d6db43fad6630376ba59451f848f4d387c \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.8.0.jar.sha1 b/server/licenses/lucene-grouping-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..ab132121b2edc --- /dev/null +++ b/server/licenses/lucene-grouping-9.8.0.jar.sha1 @@ -0,0 +1 @@ +d39184518351178c404ed9669fc6cb6111f2288d \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index c3ad03ca53b13..0000000000000 --- a/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -83ab97638bb5269f950d75bba5675d3cfb63f2fa \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.8.0.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..c7cb678fb7b72 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.8.0.jar.sha1 @@ -0,0 +1 @@ +1ac38c8278dbd63dfab30744a41dd955a415a31c \ No newline at end of file diff --git a/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index c2a4c5334b314..0000000000000 --- a/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -97c26362151908dc892263edda3872abbacb71a8 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.8.0.jar.sha1 b/server/licenses/lucene-join-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..2b6cb8af4faf6 --- /dev/null +++ b/server/licenses/lucene-join-9.8.0.jar.sha1 @@ -0,0 +1 @@ +3d64fc57bb6e718d906413a9f73c713e6d4d8bb0 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 32534d07e47dc..0000000000000 --- a/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8337eddc0dddd0d7dd50c5aa0d17e5e31592f9fa \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.8.0.jar.sha1 b/server/licenses/lucene-memory-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..5fdfee401dd0a --- /dev/null +++ b/server/licenses/lucene-memory-9.8.0.jar.sha1 @@ -0,0 +1 @@ +5283ac71d6ccecb5e00c7b52df2faec012f2625a \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 7db245cc521c7..0000000000000 --- a/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a2e3fae930295f0e2b401effe04eafc25692a414 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.8.0.jar.sha1 b/server/licenses/lucene-misc-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..cf815cba15862 --- /dev/null +++ b/server/licenses/lucene-misc-9.8.0.jar.sha1 @@ -0,0 +1 @@ +9a57b049cf51a5e9c9c1909c420f645f1b6f9a54 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index d01a6d733196e..0000000000000 --- a/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e88d8a464e6cfa345b946c9c8822ba7ee2a9159f \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.8.0.jar.sha1 b/server/licenses/lucene-queries-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..09f369ef18e12 --- /dev/null +++ b/server/licenses/lucene-queries-9.8.0.jar.sha1 @@ -0,0 +1 @@ +628db4ef46f1c6a05145bdac1d1bc4ace6341b13 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index c7b9640bad170..0000000000000 --- a/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9905790675c01e8dc24f9a5e6b9b28b879c65a52 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.8.0.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..2a42a8956b18b --- /dev/null +++ b/server/licenses/lucene-queryparser-9.8.0.jar.sha1 @@ -0,0 +1 @@ +982faf2bfa55542bf57fbadef54c19ac00f57cae \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index c4cd9e47624f8..0000000000000 --- a/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d6c8be427ec8ffc7e8233ffbf0d190d95a56cf14 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.8.0.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..64a0b07f72d29 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.8.0.jar.sha1 @@ -0,0 +1 @@ +06493dbd14d02537716822254866a94458f4d842 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index dfee145d3ea26..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11716d61288feaa692593bf699affa8de2b564c4 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..d1bcb0581435c --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 @@ -0,0 +1 @@ +9d9a731822ad6eefa1ba288a0c158d478522f165 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index c7410086ba86c..0000000000000 --- a/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a888e06c0535403b9e58a8dcddeb5e6513a4930 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.8.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..d17459cc569a9 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.8.0.jar.sha1 @@ -0,0 +1 @@ +ce752a52b2d4eac90633c7df7982e29504f99e76 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 deleted file mode 100644 index 6d8d4205f4d02..0000000000000 --- a/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -52dfc8bf135ed29f5baf0a967c1bb63dedb9a069 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.8.0.jar.sha1 b/server/licenses/lucene-suggest-9.8.0.jar.sha1 new file mode 100644 index 0000000000000..ff47b87672d2c --- /dev/null +++ b/server/licenses/lucene-suggest-9.8.0.jar.sha1 @@ -0,0 +1 @@ +f977f96f2093b7fddea6b67caa2e1c5b10edebf6 \ No newline at end of file From 6003560f944d5e63943d13c679797787458f1b5f Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 2 Oct 2023 13:04:38 -0400 Subject: [PATCH 083/388] Bump asm from 9.5 to 9.6 (#10302) Signed-off-by: Andriy Redko --- CHANGELOG.md | 9 +++++---- buildSrc/version.properties | 2 +- modules/lang-expression/licenses/asm-9.5.jar.sha1 | 1 - modules/lang-expression/licenses/asm-9.6.jar.sha1 | 1 + .../lang-expression/licenses/asm-commons-9.5.jar.sha1 | 1 - .../lang-expression/licenses/asm-commons-9.6.jar.sha1 | 1 + modules/lang-expression/licenses/asm-tree-9.5.jar.sha1 | 1 - modules/lang-expression/licenses/asm-tree-9.6.jar.sha1 | 1 + modules/lang-painless/licenses/asm-9.5.jar.sha1 | 1 - modules/lang-painless/licenses/asm-9.6.jar.sha1 | 1 + modules/lang-painless/licenses/asm-analysis-9.5.jar.sha1 | 1 - modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1 | 1 + modules/lang-painless/licenses/asm-commons-9.5.jar.sha1 | 1 - modules/lang-painless/licenses/asm-commons-9.6.jar.sha1 | 1 + modules/lang-painless/licenses/asm-tree-9.5.jar.sha1 | 1 - modules/lang-painless/licenses/asm-tree-9.6.jar.sha1 | 1 + modules/lang-painless/licenses/asm-util-9.5.jar.sha1 | 1 - modules/lang-painless/licenses/asm-util-9.6.jar.sha1 | 1 + 18 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 modules/lang-expression/licenses/asm-9.5.jar.sha1 create mode 100644 modules/lang-expression/licenses/asm-9.6.jar.sha1 delete mode 100644 modules/lang-expression/licenses/asm-commons-9.5.jar.sha1 create mode 100644 modules/lang-expression/licenses/asm-commons-9.6.jar.sha1 delete mode 100644 modules/lang-expression/licenses/asm-tree-9.5.jar.sha1 create mode 100644 modules/lang-expression/licenses/asm-tree-9.6.jar.sha1 delete mode 100644 modules/lang-painless/licenses/asm-9.5.jar.sha1 create mode 100644 modules/lang-painless/licenses/asm-9.6.jar.sha1 delete mode 100644 modules/lang-painless/licenses/asm-analysis-9.5.jar.sha1 create mode 100644 modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1 delete mode 100644 modules/lang-painless/licenses/asm-commons-9.5.jar.sha1 create mode 100644 modules/lang-painless/licenses/asm-commons-9.6.jar.sha1 delete mode 100644 modules/lang-painless/licenses/asm-tree-9.5.jar.sha1 create mode 100644 modules/lang-painless/licenses/asm-tree-9.6.jar.sha1 delete mode 100644 modules/lang-painless/licenses/asm-util-9.5.jar.sha1 create mode 100644 modules/lang-painless/licenses/asm-util-9.6.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 772198d5d0544..e1834df696769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,10 +104,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206)) - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208)) - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) -- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` -- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` -- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` -- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) +- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump Lucene from 9.7.0 to 9.8.0 ([#10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) +- Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index dae68940a7b7f..243a1b2c6f57e 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -16,7 +16,7 @@ icu4j = 70.1 supercsv = 2.4.0 log4j = 2.20.0 slf4j = 1.7.36 -asm = 9.5 +asm = 9.6 jettison = 1.5.4 woodstox = 6.4.0 kotlin = 1.7.10 diff --git a/modules/lang-expression/licenses/asm-9.5.jar.sha1 b/modules/lang-expression/licenses/asm-9.5.jar.sha1 deleted file mode 100644 index ea4aa3581dc87..0000000000000 --- a/modules/lang-expression/licenses/asm-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc6ea1875f4d64fbc85e1691c95b96a3d8569c90 \ No newline at end of file diff --git a/modules/lang-expression/licenses/asm-9.6.jar.sha1 b/modules/lang-expression/licenses/asm-9.6.jar.sha1 new file mode 100644 index 0000000000000..2d9e6a9d3cfd6 --- /dev/null +++ b/modules/lang-expression/licenses/asm-9.6.jar.sha1 @@ -0,0 +1 @@ +aa205cf0a06dbd8e04ece91c0b37c3f5d567546a \ No newline at end of file diff --git a/modules/lang-expression/licenses/asm-commons-9.5.jar.sha1 b/modules/lang-expression/licenses/asm-commons-9.5.jar.sha1 deleted file mode 100644 index 5be792660c19f..0000000000000 --- a/modules/lang-expression/licenses/asm-commons-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ab5b5800a3910d30d3a3e64fdb00fd0cb42de0 \ No newline at end of file diff --git a/modules/lang-expression/licenses/asm-commons-9.6.jar.sha1 b/modules/lang-expression/licenses/asm-commons-9.6.jar.sha1 new file mode 100644 index 0000000000000..a0814f495771f --- /dev/null +++ b/modules/lang-expression/licenses/asm-commons-9.6.jar.sha1 @@ -0,0 +1 @@ +f1a9e5508eff490744144565c47326c8648be309 \ No newline at end of file diff --git a/modules/lang-expression/licenses/asm-tree-9.5.jar.sha1 b/modules/lang-expression/licenses/asm-tree-9.5.jar.sha1 deleted file mode 100644 index fb42db6a9d15c..0000000000000 --- a/modules/lang-expression/licenses/asm-tree-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fd33c8b6373abaa675be407082fdfda35021254a \ No newline at end of file diff --git a/modules/lang-expression/licenses/asm-tree-9.6.jar.sha1 b/modules/lang-expression/licenses/asm-tree-9.6.jar.sha1 new file mode 100644 index 0000000000000..101eb03b4b736 --- /dev/null +++ b/modules/lang-expression/licenses/asm-tree-9.6.jar.sha1 @@ -0,0 +1 @@ +c0cdda9d211e965d2a4448aa3fd86110f2f8c2de \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-9.5.jar.sha1 b/modules/lang-painless/licenses/asm-9.5.jar.sha1 deleted file mode 100644 index ea4aa3581dc87..0000000000000 --- a/modules/lang-painless/licenses/asm-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc6ea1875f4d64fbc85e1691c95b96a3d8569c90 \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-9.6.jar.sha1 new file mode 100644 index 0000000000000..2d9e6a9d3cfd6 --- /dev/null +++ b/modules/lang-painless/licenses/asm-9.6.jar.sha1 @@ -0,0 +1 @@ +aa205cf0a06dbd8e04ece91c0b37c3f5d567546a \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-analysis-9.5.jar.sha1 b/modules/lang-painless/licenses/asm-analysis-9.5.jar.sha1 deleted file mode 100644 index 9e87d3ce7d719..0000000000000 --- a/modules/lang-painless/licenses/asm-analysis-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -490bacc77de7cbc0be1a30bb3471072d705be4a4 \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1 new file mode 100644 index 0000000000000..fa42ea1198165 --- /dev/null +++ b/modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1 @@ -0,0 +1 @@ +9ce6c7b174bd997fc2552dff47964546bd7a5ec3 \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-commons-9.5.jar.sha1 b/modules/lang-painless/licenses/asm-commons-9.5.jar.sha1 deleted file mode 100644 index 5be792660c19f..0000000000000 --- a/modules/lang-painless/licenses/asm-commons-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ab5b5800a3910d30d3a3e64fdb00fd0cb42de0 \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-commons-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-commons-9.6.jar.sha1 new file mode 100644 index 0000000000000..a0814f495771f --- /dev/null +++ b/modules/lang-painless/licenses/asm-commons-9.6.jar.sha1 @@ -0,0 +1 @@ +f1a9e5508eff490744144565c47326c8648be309 \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-tree-9.5.jar.sha1 b/modules/lang-painless/licenses/asm-tree-9.5.jar.sha1 deleted file mode 100644 index fb42db6a9d15c..0000000000000 --- a/modules/lang-painless/licenses/asm-tree-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fd33c8b6373abaa675be407082fdfda35021254a \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-tree-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-tree-9.6.jar.sha1 new file mode 100644 index 0000000000000..101eb03b4b736 --- /dev/null +++ b/modules/lang-painless/licenses/asm-tree-9.6.jar.sha1 @@ -0,0 +1 @@ +c0cdda9d211e965d2a4448aa3fd86110f2f8c2de \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-util-9.5.jar.sha1 b/modules/lang-painless/licenses/asm-util-9.5.jar.sha1 deleted file mode 100644 index 5fffbfe655deb..0000000000000 --- a/modules/lang-painless/licenses/asm-util-9.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64b5a1fc8c1b15ed2efd6a063e976bc8d3dc5ffe \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-util-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-util-9.6.jar.sha1 new file mode 100644 index 0000000000000..1f42ac62dc69c --- /dev/null +++ b/modules/lang-painless/licenses/asm-util-9.6.jar.sha1 @@ -0,0 +1 @@ +f77caf84eb93786a749b2baa40865b9613e3eaee \ No newline at end of file From fa66bebf88d6fc69765d5ecb26930ece94477024 Mon Sep 17 00:00:00 2001 From: Rishabh Maurya Date: Mon, 2 Oct 2023 13:31:02 -0700 Subject: [PATCH 084/388] Use of LogByteSizeMergePolicy for data stream use cases (#9992) * Configurable merge policy for index * additional setting to configure merge policy for timestamp based index * introduction of logbytesize merge policy as an option Signed-off-by: Rishabh Maurya * remove the trace log not required anymore Signed-off-by: Rishabh Maurya * Refactor the merge policy extraction logic Signed-off-by: Rishabh Maurya * Rename constant DEFAULT to DEFAULT_POLICY Signed-off-by: Rishabh Maurya * Simplify merge policy extraction and selection logic Signed-off-by: Rishabh Maurya * missing javadoc error Signed-off-by: Rishabh Maurya * Renaming log byte size policy setting with mb Signed-off-by: Rishabh Maurya * Move validation exception to enum from setting defn Signed-off-by: Rishabh Maurya * rename time_index to time_series_index Signed-off-by: Rishabh Maurya --------- Signed-off-by: Rishabh Maurya --- CHANGELOG.md | 1 + .../gateway/RecoveryFromGatewayIT.java | 4 +- .../RemoveCorruptedShardDataCommandIT.java | 4 +- .../index/store/CorruptedFileIT.java | 10 +- .../indices/stats/IndexStatsIT.java | 10 +- .../java/org/opensearch/update/UpdateIT.java | 4 +- .../common/settings/ClusterSettings.java | 1 + .../common/settings/IndexScopedSettings.java | 29 +- .../org/opensearch/index/IndexSettings.java | 156 ++++++- .../index/LogByteSizeMergePolicyProvider.java | 166 +++++++ .../opensearch/index/MergePolicyProvider.java | 31 ++ ...ig.java => TieredMergePolicyProvider.java} | 84 ++-- .../opensearch/index/shard/IndexShard.java | 2 +- .../segments/IndicesSegmentsRequestTests.java | 4 +- .../index/MergePolicySettingsTests.java | 421 +++++++++++++++--- .../index/MergeSchedulerSettingsTests.java | 8 +- .../RemoveCorruptedShardDataCommandTests.java | 4 +- .../indices/recovery/RecoveryTests.java | 4 +- .../test/OpenSearchIntegTestCase.java | 4 +- 19 files changed, 776 insertions(+), 171 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/LogByteSizeMergePolicyProvider.java create mode 100644 server/src/main/java/org/opensearch/index/MergePolicyProvider.java rename server/src/main/java/org/opensearch/index/{MergePolicyConfig.java => TieredMergePolicyProvider.java} (82%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1834df696769..7519e242c0acf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679)) - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110)) - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618)) +- Configurable merge policy for index with an option to choose from LogByteSize and Tiered merge policy ([#9992](https://github.com/opensearch-project/OpenSearch/pull/9992)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java index 2bab61f3e1c4c..229cd7bffad2f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java @@ -53,7 +53,7 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.index.engine.Engine; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.shard.ShardPath; @@ -519,7 +519,7 @@ public void testReuseInFileBasedPeerRecovery() throws Exception { .put("number_of_replicas", 1) // disable merges to keep segments the same - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) // expire retention leases quickly .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java index f8c2acbf99f70..b431079476624 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -73,7 +73,7 @@ import org.opensearch.env.TestEnvironment; import org.opensearch.gateway.GatewayMetaState; import org.opensearch.index.IndexSettings; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.index.MockEngineFactoryPlugin; import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.translog.TestTranslog; @@ -135,7 +135,7 @@ public void testCorruptIndex() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1") .put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), "checksum") diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java index 7e1d0792e3ddb..8291fef5d177b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java @@ -72,7 +72,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; @@ -167,7 +167,7 @@ public void testCorruptFileAndRecover() throws ExecutionException, InterruptedEx Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) // no checkindex - we corrupt shards on purpose .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no translog based flush - it might change the .liv / segments.N files @@ -286,7 +286,7 @@ public void testCorruptPrimaryNoReplica() throws ExecutionException, Interrupted prepareCreate("test").setSettings( Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0") - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on // purpose // no translog based flush - it might change the .liv / segments.N files @@ -552,7 +552,7 @@ public void testCorruptFileThenSnapshotAndRestore() throws ExecutionException, I prepareCreate("test").setSettings( Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0") // no replicas for this test - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) // no checkindex - we corrupt shards on purpose .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no translog based flush - it might change the .liv / segments.N files @@ -624,7 +624,7 @@ public void testReplicaCorruption() throws Exception { prepareCreate("test").setSettings( Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1) - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on // purpose .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java index a0f01acd1f8e9..0967acb37d3e8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java @@ -66,8 +66,8 @@ import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; -import org.opensearch.index.MergePolicyConfig; import org.opensearch.index.MergeSchedulerConfig; +import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.VersionType; import org.opensearch.index.cache.query.QueryCacheStats; import org.opensearch.index.engine.VersionConflictEngineException; @@ -589,8 +589,8 @@ public void testNonThrottleStats() throws Exception { prepareCreate("test").setSettings( settingsBuilder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1") .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "10000") ) @@ -621,8 +621,8 @@ public void testThrottleStats() throws Exception { prepareCreate("test").setSettings( settingsBuilder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1") .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "1") .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC.name()) diff --git a/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java index 442268d513fc3..b46d27bafb2a5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/update/UpdateIT.java @@ -50,7 +50,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.plugins.Plugin; @@ -669,7 +669,7 @@ public void run() { public void testStressUpdateDeleteConcurrency() throws Exception { // We create an index with merging disabled so that deletes don't get merged away - assertAcked(prepareCreate("test").setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false))); + assertAcked(prepareCreate("test").setSettings(Settings.builder().put(MergePolicyProvider.INDEX_MERGE_ENABLED, false))); ensureGreen(); Script fieldIncScript = new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field")); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 032027384f106..5261d40387dc6 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -457,6 +457,7 @@ public void apply(Settings value, Settings current, Settings previous) { NetworkService.TCP_CONNECT_TIMEOUT, IndexSettings.QUERY_STRING_ANALYZE_WILDCARD, IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD, + IndexSettings.TIME_SERIES_INDEX_MERGE_POLICY, ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING, ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING, diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 5b2afc44600bd..83bf8c82ee3dd 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -45,9 +45,11 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.IndexSortConfig; import org.opensearch.index.IndexingSlowLog; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.LogByteSizeMergePolicyProvider; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.index.MergeSchedulerConfig; import org.opensearch.index.SearchSlowLog; +import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.fielddata.IndexFieldDataService; @@ -120,14 +122,14 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexingSlowLog.INDEX_INDEXING_SLOWLOG_LEVEL_SETTING, IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING, - MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, + TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING, + TieredMergePolicyProvider.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING, + TieredMergePolicyProvider.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, + TieredMergePolicyProvider.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING, + TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, + TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, + TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, + TieredMergePolicyProvider.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, IndexSortConfig.INDEX_SORT_FIELD_SETTING, IndexSortConfig.INDEX_SORT_ORDER_SETTING, IndexSortConfig.INDEX_SORT_MISSING_SETTING, @@ -202,6 +204,13 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.INDEX_MERGE_ON_FLUSH_ENABLED, IndexSettings.INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME, IndexSettings.INDEX_MERGE_ON_FLUSH_POLICY, + IndexSettings.INDEX_MERGE_POLICY, + LogByteSizeMergePolicyProvider.INDEX_LBS_MERGE_POLICY_MERGE_FACTOR_SETTING, + LogByteSizeMergePolicyProvider.INDEX_LBS_MERGE_POLICY_MIN_MERGE_SETTING, + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGE_SEGMENT_SETTING, + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGE_SEGMENT_FOR_FORCED_MERGE_SETTING, + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGED_DOCS_SETTING, + LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING, IndexSettings.DEFAULT_SEARCH_PIPELINE, // Settings for Searchable Snapshots @@ -275,7 +284,7 @@ public boolean isPrivateSetting(String key) { case IndexMetadata.SETTING_HISTORY_UUID: case IndexMetadata.SETTING_VERSION_UPGRADED: case IndexMetadata.SETTING_INDEX_PROVIDED_NAME: - case MergePolicyConfig.INDEX_MERGE_ENABLED: + case MergePolicyProvider.INDEX_MERGE_ENABLED: // we keep the shrink settings for BWC - this can be removed in 8.0 // we can't remove in 7 since this setting might be baked into an index coming in via a full cluster restart from 6.0 case "index.shrink.source.uuid": diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 1e4224c314f05..ce6c1a5ad6284 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -54,6 +54,7 @@ import org.opensearch.node.Node; import org.opensearch.search.pipeline.SearchPipelineService; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -83,9 +84,42 @@ */ @PublicApi(since = "1.0.0") public final class IndexSettings { - private static final String MERGE_ON_FLUSH_DEFAULT_POLICY = "default"; + private static final String DEFAULT_POLICY = "default"; private static final String MERGE_ON_FLUSH_MERGE_POLICY = "merge-on-flush"; + /** + * Enum representing supported merge policies + */ + public enum IndexMergePolicy { + TIERED("tiered"), + LOG_BYTE_SIZE("log_byte_size"), + DEFAULT_POLICY(IndexSettings.DEFAULT_POLICY); + + private final String value; + + IndexMergePolicy(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static IndexMergePolicy fromString(String text) { + for (IndexMergePolicy policy : IndexMergePolicy.values()) { + if (policy.value.equals(text)) { + return policy; + } + } + throw new IllegalArgumentException( + "The setting has unsupported policy specified: " + + text + + ". Please use one of: " + + String.join(", ", Arrays.stream(IndexMergePolicy.values()).map(IndexMergePolicy::getValue).toArray(String[]::new)) + ); + } + } + public static final Setting> DEFAULT_FIELD_SETTING = Setting.listSetting( "index.query.default_field", Collections.singletonList("*"), @@ -566,11 +600,25 @@ public final class IndexSettings { public static final Setting INDEX_MERGE_ON_FLUSH_POLICY = Setting.simpleString( "index.merge_on_flush.policy", - MERGE_ON_FLUSH_DEFAULT_POLICY, + DEFAULT_POLICY, Property.IndexScope, Property.Dynamic ); + public static final Setting INDEX_MERGE_POLICY = Setting.simpleString( + "index.merge.policy", + DEFAULT_POLICY, + IndexMergePolicy::fromString, + Property.IndexScope + ); + + public static final Setting TIME_SERIES_INDEX_MERGE_POLICY = Setting.simpleString( + "indices.time_series_index.default_index_merge_policy", + DEFAULT_POLICY, + IndexMergePolicy::fromString, + Property.NodeScope + ); + public static final Setting SEARCHABLE_SNAPSHOT_REPOSITORY = Setting.simpleString( "index.searchable_snapshot.repository", Property.IndexScope, @@ -651,7 +699,8 @@ public final class IndexSettings { private volatile ByteSizeValue generationThresholdSize; private volatile ByteSizeValue flushAfterMergeThresholdSize; private final MergeSchedulerConfig mergeSchedulerConfig; - private final MergePolicyConfig mergePolicyConfig; + private final TieredMergePolicyProvider tieredMergePolicyProvider; + private final LogByteSizeMergePolicyProvider logByteSizeMergePolicyProvider; private final IndexSortConfig indexSortConfig; private final IndexScopedSettings scopedSettings; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); @@ -844,7 +893,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti maxAnalyzedOffset = scopedSettings.get(MAX_ANALYZED_OFFSET_SETTING); maxTermsCount = scopedSettings.get(MAX_TERMS_COUNT_SETTING); maxRegexLength = scopedSettings.get(MAX_REGEX_LENGTH_SETTING); - this.mergePolicyConfig = new MergePolicyConfig(logger, this); + this.tieredMergePolicyProvider = new TieredMergePolicyProvider(logger, this); + this.logByteSizeMergePolicyProvider = new LogByteSizeMergePolicyProvider(logger, this); this.indexSortConfig = new IndexSortConfig(this); searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE); @@ -866,33 +916,59 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti * Now this sortField (IndexSort) is stored in SegmentInfo and we need to maintain backward compatibility for them. */ widenIndexSortType = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).before(V_2_7_0); - - scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio); scopedSettings.addSettingsUpdateConsumer( - MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING, - mergePolicyConfig::setDeletesPctAllowed + TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING, + tieredMergePolicyProvider::setNoCFSRatio ); scopedSettings.addSettingsUpdateConsumer( - MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, - mergePolicyConfig::setExpungeDeletesAllowed + TieredMergePolicyProvider.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING, + tieredMergePolicyProvider::setDeletesPctAllowed ); scopedSettings.addSettingsUpdateConsumer( - MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING, - mergePolicyConfig::setFloorSegmentSetting + TieredMergePolicyProvider.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, + tieredMergePolicyProvider::setExpungeDeletesAllowed ); scopedSettings.addSettingsUpdateConsumer( - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, - mergePolicyConfig::setMaxMergesAtOnce + TieredMergePolicyProvider.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING, + tieredMergePolicyProvider::setFloorSegmentSetting ); scopedSettings.addSettingsUpdateConsumer( - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, - mergePolicyConfig::setMaxMergedSegment + TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, + tieredMergePolicyProvider::setMaxMergesAtOnce ); scopedSettings.addSettingsUpdateConsumer( - MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, - mergePolicyConfig::setSegmentsPerTier + TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, + tieredMergePolicyProvider::setMaxMergedSegment + ); + scopedSettings.addSettingsUpdateConsumer( + TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, + tieredMergePolicyProvider::setSegmentsPerTier ); + scopedSettings.addSettingsUpdateConsumer( + LogByteSizeMergePolicyProvider.INDEX_LBS_MERGE_POLICY_MERGE_FACTOR_SETTING, + logByteSizeMergePolicyProvider::setLBSMergeFactor + ); + scopedSettings.addSettingsUpdateConsumer( + LogByteSizeMergePolicyProvider.INDEX_LBS_MERGE_POLICY_MIN_MERGE_SETTING, + logByteSizeMergePolicyProvider::setLBSMinMergedMB + ); + scopedSettings.addSettingsUpdateConsumer( + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGE_SEGMENT_SETTING, + logByteSizeMergePolicyProvider::setLBSMaxMergeSegment + ); + scopedSettings.addSettingsUpdateConsumer( + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGE_SEGMENT_FOR_FORCED_MERGE_SETTING, + logByteSizeMergePolicyProvider::setLBSMaxMergeMBForForcedMerge + ); + scopedSettings.addSettingsUpdateConsumer( + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGED_DOCS_SETTING, + logByteSizeMergePolicyProvider::setLBSMaxMergeDocs + ); + scopedSettings.addSettingsUpdateConsumer( + LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING, + logByteSizeMergePolicyProvider::setLBSNoCFSRatio + ); scopedSettings.addSettingsUpdateConsumer( MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, @@ -1439,9 +1515,43 @@ public long getGcDeletesInMillis() { /** * Returns the merge policy that should be used for this index. - */ - public MergePolicy getMergePolicy() { - return mergePolicyConfig.getMergePolicy(); + * @param isTimeSeriesIndex true if index contains @timestamp field + */ + public MergePolicy getMergePolicy(boolean isTimeSeriesIndex) { + String indexScopedPolicy = scopedSettings.get(INDEX_MERGE_POLICY); + MergePolicyProvider mergePolicyProvider = null; + IndexMergePolicy indexMergePolicy = IndexMergePolicy.fromString(indexScopedPolicy); + switch (indexMergePolicy) { + case TIERED: + mergePolicyProvider = tieredMergePolicyProvider; + break; + case LOG_BYTE_SIZE: + mergePolicyProvider = logByteSizeMergePolicyProvider; + break; + case DEFAULT_POLICY: + if (isTimeSeriesIndex) { + String nodeScopedTimeSeriesIndexPolicy = TIME_SERIES_INDEX_MERGE_POLICY.get(nodeSettings); + IndexMergePolicy nodeMergePolicy = IndexMergePolicy.fromString(nodeScopedTimeSeriesIndexPolicy); + switch (nodeMergePolicy) { + case TIERED: + case DEFAULT_POLICY: + mergePolicyProvider = tieredMergePolicyProvider; + break; + case LOG_BYTE_SIZE: + mergePolicyProvider = logByteSizeMergePolicyProvider; + break; + } + } else { + mergePolicyProvider = tieredMergePolicyProvider; + } + break; + } + assert mergePolicyProvider != null : "should not happen as validation for invalid merge policy values " + + "are part of setting definition"; + if (logger.isTraceEnabled()) { + logger.trace("Index: " + this.index.getName() + ", Merge policy used: " + mergePolicyProvider); + } + return mergePolicyProvider.getMergePolicy(); } public T getValue(Setting setting) { @@ -1632,7 +1742,7 @@ public boolean isMergeOnFlushEnabled() { } private void setMergeOnFlushPolicy(String policy) { - if (Strings.isEmpty(policy) || MERGE_ON_FLUSH_DEFAULT_POLICY.equalsIgnoreCase(policy)) { + if (Strings.isEmpty(policy) || DEFAULT_POLICY.equalsIgnoreCase(policy)) { mergeOnFlushPolicy = null; } else if (MERGE_ON_FLUSH_MERGE_POLICY.equalsIgnoreCase(policy)) { this.mergeOnFlushPolicy = MergeOnFlushMergePolicy::new; @@ -1643,7 +1753,7 @@ private void setMergeOnFlushPolicy(String policy) { + " has unsupported policy specified: " + policy + ". Please use one of: " - + MERGE_ON_FLUSH_DEFAULT_POLICY + + DEFAULT_POLICY + ", " + MERGE_ON_FLUSH_MERGE_POLICY ); diff --git a/server/src/main/java/org/opensearch/index/LogByteSizeMergePolicyProvider.java b/server/src/main/java/org/opensearch/index/LogByteSizeMergePolicyProvider.java new file mode 100644 index 0000000000000..0b762d781957c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/LogByteSizeMergePolicyProvider.java @@ -0,0 +1,166 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.LogByteSizeMergePolicy; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.NoMergePolicy; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; + +import static org.apache.lucene.index.LogMergePolicy.DEFAULT_MAX_MERGE_DOCS; +import static org.apache.lucene.index.LogMergePolicy.DEFAULT_NO_CFS_RATIO; + +/** + *

+ * The LogByteSizeMergePolicy is an alternative merge policy primarily used here to optimize the merging of segments in scenarios + * with index with timestamps. + * While the TieredMergePolicy is the default choice, the LogByteSizeMergePolicy can be configured + * as the default merge policy for time-index data using the index.datastream_merge.policy setting. + * + *

+ * Unlike the TieredMergePolicy, which prioritizes merging segments of equal sizes, the LogByteSizeMergePolicy + * specializes in merging adjacent segments efficiently. + * This characteristic makes it particularly well-suited for range queries on time-index data. + * Typically, adjacent segments in time-index data often contain documents with similar timestamps. + * When these segments are merged, the resulting segment covers a range of timestamps with reduced overlap compared + * to the adjacent segments. This reduced overlap remains even as segments grow older and larger, + * which can significantly benefit range queries on timestamps. + * + *

+ * In contrast, the TieredMergePolicy does not honor this timestamp range optimization. It focuses on merging segments + * of equal sizes and does not consider adjacency. Consequently, as segments grow older and larger, + * the overlap of timestamp ranges among adjacent segments managed by TieredMergePolicy can increase. + * This can lead to inefficiencies in range queries on timestamps, as the number of segments to be scanned + * within a given timestamp range could become high. + * + * @opensearch.internal + */ +public class LogByteSizeMergePolicyProvider implements MergePolicyProvider { + private final LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy(); + + private final Logger logger; + private final boolean mergesEnabled; + + public static final ByteSizeValue DEFAULT_MIN_MERGE = new ByteSizeValue(2, ByteSizeUnit.MB); + public static final int DEFAULT_MERGE_FACTOR = 10; + + public static final ByteSizeValue DEFAULT_MAX_MERGED_SEGMENT = new ByteSizeValue(5, ByteSizeUnit.GB); + + public static final ByteSizeValue DEFAULT_MAX_MERGE_SEGMENT_FORCE_MERGE = new ByteSizeValue(Long.MAX_VALUE); + + public static final Setting INDEX_LBS_MERGE_POLICY_MERGE_FACTOR_SETTING = Setting.intSetting( + "index.merge.log_byte_size_policy.merge_factor", + DEFAULT_MERGE_FACTOR, // keeping it same as default max merge at once for tiered merge policy + 2, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + public static final Setting INDEX_LBS_MERGE_POLICY_MIN_MERGE_SETTING = Setting.byteSizeSetting( + "index.merge.log_byte_size_policy.min_merge", + DEFAULT_MIN_MERGE, // keeping it same as default floor segment for tiered merge policy + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + public static final Setting INDEX_LBS_MAX_MERGE_SEGMENT_SETTING = Setting.byteSizeSetting( + "index.merge.log_byte_size_policy.max_merge_segment", + DEFAULT_MAX_MERGED_SEGMENT, // keeping default same as tiered merge policy + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + public static final Setting INDEX_LBS_MAX_MERGE_SEGMENT_FOR_FORCED_MERGE_SETTING = Setting.byteSizeSetting( + "index.merge.log_byte_size_policy.max_merge_segment_forced_merge", + DEFAULT_MAX_MERGE_SEGMENT_FORCE_MERGE, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + public static final Setting INDEX_LBS_MAX_MERGED_DOCS_SETTING = Setting.intSetting( + "index.merge.log_byte_size_policy.max_merged_docs", + DEFAULT_MAX_MERGE_DOCS, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + public static final Setting INDEX_LBS_NO_CFS_RATIO_SETTING = new Setting<>( + "index.merge.log_byte_size_policy.no_cfs_ratio", + Double.toString(DEFAULT_NO_CFS_RATIO), + TieredMergePolicyProvider::parseNoCFSRatio, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + LogByteSizeMergePolicyProvider(Logger logger, IndexSettings indexSettings) { + this.logger = logger; + this.mergesEnabled = indexSettings.getSettings().getAsBoolean(INDEX_MERGE_ENABLED, true); + + // Undocumented settings, works great with defaults + logByteSizeMergePolicy.setMergeFactor(indexSettings.getValue(INDEX_LBS_MERGE_POLICY_MERGE_FACTOR_SETTING)); + logByteSizeMergePolicy.setMinMergeMB(indexSettings.getValue(INDEX_LBS_MERGE_POLICY_MIN_MERGE_SETTING).getMbFrac()); + logByteSizeMergePolicy.setMaxMergeMB(indexSettings.getValue(INDEX_LBS_MAX_MERGE_SEGMENT_SETTING).getMbFrac()); + logByteSizeMergePolicy.setMaxMergeMBForForcedMerge( + indexSettings.getValue(INDEX_LBS_MAX_MERGE_SEGMENT_FOR_FORCED_MERGE_SETTING).getMbFrac() + ); + logByteSizeMergePolicy.setMaxMergeDocs(indexSettings.getValue(INDEX_LBS_MAX_MERGED_DOCS_SETTING)); + logByteSizeMergePolicy.setNoCFSRatio(indexSettings.getValue(INDEX_LBS_NO_CFS_RATIO_SETTING)); + } + + @Override + public MergePolicy getMergePolicy() { + return mergesEnabled ? logByteSizeMergePolicy : NoMergePolicy.INSTANCE; + } + + void setLBSMergeFactor(int mergeFactor) { + logByteSizeMergePolicy.setMergeFactor(mergeFactor); + } + + void setLBSMaxMergeSegment(ByteSizeValue maxMergeSegment) { + logByteSizeMergePolicy.setMaxMergeMB(maxMergeSegment.getMbFrac()); + } + + void setLBSMinMergedMB(ByteSizeValue minMergedSize) { + logByteSizeMergePolicy.setMinMergeMB(minMergedSize.getMbFrac()); + } + + void setLBSMaxMergeMBForForcedMerge(ByteSizeValue maxMergeForcedMerge) { + logByteSizeMergePolicy.setMaxMergeMBForForcedMerge(maxMergeForcedMerge.getMbFrac()); + } + + void setLBSMaxMergeDocs(int maxMergeDocs) { + logByteSizeMergePolicy.setMaxMergeDocs(maxMergeDocs); + } + + void setLBSNoCFSRatio(Double noCFSRatio) { + logByteSizeMergePolicy.setNoCFSRatio(noCFSRatio); + } + + @Override + public String toString() { + return "LogByteSizeMergePolicyProvider{" + + "mergeFactor=" + + logByteSizeMergePolicy.getMergeFactor() + + ", minMergeMB=" + + logByteSizeMergePolicy.getMinMergeMB() + + ", maxMergeMB=" + + logByteSizeMergePolicy.getMaxMergeMB() + + ", maxMergeMBForForcedMerge=" + + logByteSizeMergePolicy.getMaxMergeMBForForcedMerge() + + ", maxMergedDocs=" + + logByteSizeMergePolicy.getMaxMergeDocs() + + ", noCFSRatio=" + + logByteSizeMergePolicy.getNoCFSRatio() + + '}'; + } + +} diff --git a/server/src/main/java/org/opensearch/index/MergePolicyProvider.java b/server/src/main/java/org/opensearch/index/MergePolicyProvider.java new file mode 100644 index 0000000000000..6f734314f758f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/MergePolicyProvider.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index; + +import org.apache.lucene.index.MergePolicy; +import org.opensearch.common.annotation.InternalApi; + +/** + * A provider for obtaining merge policies used by OpenSearch indexes. + * + * @opensearch.internal + */ + +@InternalApi +public interface MergePolicyProvider { + // don't convert to Setting<> and register... we only set this in tests and register via a plugin + String INDEX_MERGE_ENABLED = "index.merge.enabled"; + + /** + * Gets the merge policy to be used for index. + * + * @return The merge policy instance. + */ + MergePolicy getMergePolicy(); +} diff --git a/server/src/main/java/org/opensearch/index/MergePolicyConfig.java b/server/src/main/java/org/opensearch/index/TieredMergePolicyProvider.java similarity index 82% rename from server/src/main/java/org/opensearch/index/MergePolicyConfig.java rename to server/src/main/java/org/opensearch/index/TieredMergePolicyProvider.java index fe2af21dfe039..d5d354c6c960a 100644 --- a/server/src/main/java/org/opensearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/opensearch/index/TieredMergePolicyProvider.java @@ -33,6 +33,7 @@ package org.opensearch.index; import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.LogByteSizeMergePolicy; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.TieredMergePolicy; @@ -47,9 +48,12 @@ * where the index data is stored, and are immutable up to delete markers. * Segments are, periodically, merged into larger segments to keep the * index size at bay and expunge deletes. + * This class customizes and exposes 2 merge policies from lucene - + * {@link LogByteSizeMergePolicy} and {@link TieredMergePolicy}. + * * *

- * Merges select segments of approximately equal size, subject to an allowed + * Tiered merge policy select segments of approximately equal size, subject to an allowed * number of segments per tier. The merge policy is able to merge * non-adjacent segments, and separates how many segments are merged at once from how many * segments are allowed per tier. It also does not over-merge (i.e., cascade merges). @@ -125,8 +129,9 @@ * @opensearch.internal */ -public final class MergePolicyConfig { - private final OpenSearchTieredMergePolicy mergePolicy = new OpenSearchTieredMergePolicy(); +public final class TieredMergePolicyProvider implements MergePolicyProvider { + private final OpenSearchTieredMergePolicy tieredMergePolicy = new OpenSearchTieredMergePolicy(); + private final Logger logger; private final boolean mergesEnabled; @@ -137,10 +142,11 @@ public final class MergePolicyConfig { public static final double DEFAULT_SEGMENTS_PER_TIER = 10.0d; public static final double DEFAULT_RECLAIM_DELETES_WEIGHT = 2.0d; public static final double DEFAULT_DELETES_PCT_ALLOWED = 20.0d; + public static final Setting INDEX_COMPOUND_FORMAT_SETTING = new Setting<>( "index.compound_format", Double.toString(TieredMergePolicy.DEFAULT_NO_CFS_RATIO), - MergePolicyConfig::parseNoCFSRatio, + TieredMergePolicyProvider::parseNoCFSRatio, Property.Dynamic, Property.IndexScope ); @@ -194,10 +200,8 @@ public final class MergePolicyConfig { Property.Dynamic, Property.IndexScope ); - // don't convert to Setting<> and register... we only set this in tests and register via a plugin - public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; - MergePolicyConfig(Logger logger, IndexSettings indexSettings) { + TieredMergePolicyProvider(Logger logger, IndexSettings indexSettings) { this.logger = logger; double forceMergeDeletesPctAllowed = indexSettings.getValue(INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING); // percentage ByteSizeValue floorSegment = indexSettings.getValue(INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING); @@ -216,54 +220,41 @@ public final class MergePolicyConfig { ); } maxMergeAtOnce = adjustMaxMergeAtOnceIfNeeded(maxMergeAtOnce, segmentsPerTier); - mergePolicy.setNoCFSRatio(indexSettings.getValue(INDEX_COMPOUND_FORMAT_SETTING)); - mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); - mergePolicy.setFloorSegmentMB(floorSegment.getMbFrac()); - mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); - mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); - mergePolicy.setSegmentsPerTier(segmentsPerTier); - mergePolicy.setDeletesPctAllowed(deletesPctAllowed); - if (logger.isTraceEnabled()) { - logger.trace( - "using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}]," - + " max_merge_at_once[{}], max_merged_segment[{}], segments_per_tier[{}]," - + " deletes_pct_allowed[{}]", - forceMergeDeletesPctAllowed, - floorSegment, - maxMergeAtOnce, - maxMergedSegment, - segmentsPerTier, - deletesPctAllowed - ); - } + tieredMergePolicy.setNoCFSRatio(indexSettings.getValue(INDEX_COMPOUND_FORMAT_SETTING)); + tieredMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); + tieredMergePolicy.setFloorSegmentMB(floorSegment.getMbFrac()); + tieredMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); + tieredMergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); + tieredMergePolicy.setSegmentsPerTier(segmentsPerTier); + tieredMergePolicy.setDeletesPctAllowed(deletesPctAllowed); } void setSegmentsPerTier(Double segmentsPerTier) { - mergePolicy.setSegmentsPerTier(segmentsPerTier); + tieredMergePolicy.setSegmentsPerTier(segmentsPerTier); } void setMaxMergedSegment(ByteSizeValue maxMergedSegment) { - mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); + tieredMergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); } void setMaxMergesAtOnce(Integer maxMergeAtOnce) { - mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); + tieredMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); } void setFloorSegmentSetting(ByteSizeValue floorSegementSetting) { - mergePolicy.setFloorSegmentMB(floorSegementSetting.getMbFrac()); + tieredMergePolicy.setFloorSegmentMB(floorSegementSetting.getMbFrac()); } void setExpungeDeletesAllowed(Double value) { - mergePolicy.setForceMergeDeletesPctAllowed(value); + tieredMergePolicy.setForceMergeDeletesPctAllowed(value); } void setNoCFSRatio(Double noCFSRatio) { - mergePolicy.setNoCFSRatio(noCFSRatio); + tieredMergePolicy.setNoCFSRatio(noCFSRatio); } void setDeletesPctAllowed(Double deletesPctAllowed) { - mergePolicy.setDeletesPctAllowed(deletesPctAllowed); + tieredMergePolicy.setDeletesPctAllowed(deletesPctAllowed); } private int adjustMaxMergeAtOnceIfNeeded(int maxMergeAtOnce, double segmentsPerTier) { @@ -285,11 +276,11 @@ private int adjustMaxMergeAtOnceIfNeeded(int maxMergeAtOnce, double segmentsPerT return maxMergeAtOnce; } - MergePolicy getMergePolicy() { - return mergesEnabled ? mergePolicy : NoMergePolicy.INSTANCE; + public MergePolicy getMergePolicy() { + return mergesEnabled ? tieredMergePolicy : NoMergePolicy.INSTANCE; } - private static double parseNoCFSRatio(String noCFSRatio) { + public static double parseNoCFSRatio(String noCFSRatio) { noCFSRatio = noCFSRatio.trim(); if (noCFSRatio.equalsIgnoreCase("true")) { return 1.0d; @@ -310,4 +301,23 @@ private static double parseNoCFSRatio(String noCFSRatio) { } } } + + @Override + public String toString() { + return "TieredMergePolicyProvider{" + + "expungeDeletesAllowed=" + + tieredMergePolicy.getForceMergeDeletesPctAllowed() + + ", floorSegment=" + + tieredMergePolicy.getFloorSegmentMB() + + ", maxMergeAtOnce=" + + tieredMergePolicy.getMaxMergeAtOnce() + + ", maxMergedSegment=" + + tieredMergePolicy.getMaxMergedSegmentMB() + + ", segmentsPerTier=" + + tieredMergePolicy.getSegmentsPerTier() + + ", deletesPctAllowed=" + + tieredMergePolicy.getDeletesPctAllowed() + + '}'; + } + } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index d476e8b7c9288..5ce066b156775 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -3772,7 +3772,7 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro indexSettings, warmer, store, - indexSettings.getMergePolicy(), + indexSettings.getMergePolicy(isTimeSeriesIndex), mapperService != null ? mapperService.indexAnalyzer() : null, similarityService.similarity(mapperService), engineConfigFactory.newCodecServiceOrDefault(indexSettings, mapperService, logger, codecService), diff --git a/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java index 67846efab2af8..d35c821b41aa0 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.settings.Settings; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.indices.IndexClosedException; import org.opensearch.plugins.Plugin; import org.opensearch.test.InternalSettingsPlugin; @@ -56,7 +56,7 @@ protected Collection> getPlugins() { public void setupIndex() { Settings settings = Settings.builder() // don't allow any merges so that the num docs is the expected segments - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) .build(); createIndex("test", settings); diff --git a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java index 387997892ee30..32c4c048d77ba 100644 --- a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java +++ b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java @@ -31,6 +31,7 @@ package org.opensearch.index; +import org.apache.lucene.index.LogByteSizeMergePolicy; import org.apache.lucene.index.NoMergePolicy; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; @@ -49,17 +50,17 @@ public class MergePolicySettingsTests extends OpenSearchTestCase { protected final ShardId shardId = new ShardId("index", "_na_", 1); public void testCompoundFileSettings() throws IOException { - assertThat(new MergePolicyConfig(logger, indexSettings(Settings.EMPTY)).getMergePolicy().getNoCFSRatio(), equalTo(0.1)); - assertThat(new MergePolicyConfig(logger, indexSettings(build(true))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build(0.5))).getMergePolicy().getNoCFSRatio(), equalTo(0.5)); - assertThat(new MergePolicyConfig(logger, indexSettings(build(1.0))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build("true"))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build("True"))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build("False"))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build("false"))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build(false))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build(0))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); - assertThat(new MergePolicyConfig(logger, indexSettings(build(0.0))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(Settings.EMPTY)).getMergePolicy().getNoCFSRatio(), equalTo(0.1)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build(true))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build(0.5))).getMergePolicy().getNoCFSRatio(), equalTo(0.5)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build(1.0))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build("true"))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build("True"))).getMergePolicy().getNoCFSRatio(), equalTo(1.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build("False"))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build("false"))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build(false))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build(0))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); + assertThat(new TieredMergePolicyProvider(logger, indexSettings(build(0.0))).getMergePolicy().getNoCFSRatio(), equalTo(0.0)); } private static IndexSettings indexSettings(Settings settings) { @@ -67,33 +68,197 @@ private static IndexSettings indexSettings(Settings settings) { } public void testNoMerges() { - MergePolicyConfig mp = new MergePolicyConfig( + TieredMergePolicyProvider tmp = new TieredMergePolicyProvider( logger, - indexSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).build()) + indexSettings(Settings.builder().put(MergePolicyProvider.INDEX_MERGE_ENABLED, false).build()) ); - assertTrue(mp.getMergePolicy() instanceof NoMergePolicy); + LogByteSizeMergePolicyProvider lbsmp = new LogByteSizeMergePolicyProvider( + logger, + indexSettings(Settings.builder().put(MergePolicyProvider.INDEX_MERGE_ENABLED, false).build()) + ); + assertTrue(tmp.getMergePolicy() instanceof NoMergePolicy); + assertTrue(lbsmp.getMergePolicy() instanceof NoMergePolicy); } public void testUpdateSettings() throws IOException { - IndexSettings indexSettings = indexSettings(EMPTY_SETTINGS); - assertThat(indexSettings.getMergePolicy().getNoCFSRatio(), equalTo(0.1)); + Settings settings = Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.TIERED.getValue()) + .build(); + IndexSettings indexSettings = indexSettings(settings); + assertThat(indexSettings.getMergePolicy(false).getNoCFSRatio(), equalTo(0.1)); indexSettings = indexSettings(build(0.9)); - assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.9)); + assertThat((indexSettings.getMergePolicy(false)).getNoCFSRatio(), equalTo(0.9)); indexSettings.updateIndexMetadata(newIndexMeta("index", build(0.1))); - assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.1)); + assertThat((indexSettings.getMergePolicy(false)).getNoCFSRatio(), equalTo(0.1)); indexSettings.updateIndexMetadata(newIndexMeta("index", build(0.0))); - assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.0)); + assertThat((indexSettings.getMergePolicy(false)).getNoCFSRatio(), equalTo(0.0)); indexSettings.updateIndexMetadata(newIndexMeta("index", build("true"))); - assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(1.0)); + assertThat((indexSettings.getMergePolicy(false)).getNoCFSRatio(), equalTo(1.0)); indexSettings.updateIndexMetadata(newIndexMeta("index", build("false"))); - assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.0)); + assertThat((indexSettings.getMergePolicy(false)).getNoCFSRatio(), equalTo(0.0)); + } + + public void testDefaultMergePolicy() throws IOException { + IndexSettings indexSettings = indexSettings(EMPTY_SETTINGS); + assertTrue(indexSettings.getMergePolicy(false) instanceof OpenSearchTieredMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof OpenSearchTieredMergePolicy); + } + + public void testMergePolicyPrecedence() throws IOException { + // 1. INDEX_MERGE_POLICY is not set + // assert defaults + IndexSettings indexSettings = indexSettings(EMPTY_SETTINGS); + assertTrue(indexSettings.getMergePolicy(false) instanceof OpenSearchTieredMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof OpenSearchTieredMergePolicy); + + // 1.1 node setting TIME_SERIES_INDEX_MERGE_POLICY is set as log_byte_size + // assert index policy is tiered whereas time series index policy is log_byte_size + Settings nodeSettings = Settings.builder() + .put(IndexSettings.TIME_SERIES_INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.LOG_BYTE_SIZE.getValue()) + .build(); + indexSettings = new IndexSettings(newIndexMeta("test", Settings.EMPTY), nodeSettings); + assertTrue(indexSettings.getMergePolicy(false) instanceof OpenSearchTieredMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof LogByteSizeMergePolicy); + + // 1.2 node setting TIME_SERIES_INDEX_MERGE_POLICY is set as tiered + // assert both index and time series index policy is tiered + nodeSettings = Settings.builder() + .put(IndexSettings.TIME_SERIES_INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.TIERED.getValue()) + .build(); + indexSettings = new IndexSettings(newIndexMeta("test", Settings.EMPTY), nodeSettings); + assertTrue(indexSettings.getMergePolicy(false) instanceof OpenSearchTieredMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof OpenSearchTieredMergePolicy); + + // 2. INDEX_MERGE_POLICY set as tiered + // assert both index and time-series-index merge policy is set as tiered + indexSettings = indexSettings( + Settings.builder().put(IndexSettings.INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.TIERED.getValue()).build() + ); + assertTrue(indexSettings.getMergePolicy(false) instanceof OpenSearchTieredMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof OpenSearchTieredMergePolicy); + + // 2.1 node setting TIME_SERIES_INDEX_MERGE_POLICY is set as log_byte_size + // assert both index and time-series-index merge policy is set as tiered + nodeSettings = Settings.builder() + .put(IndexSettings.TIME_SERIES_INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.LOG_BYTE_SIZE.getValue()) + .build(); + indexSettings = new IndexSettings( + newIndexMeta( + "test", + Settings.builder().put(IndexSettings.INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.TIERED.getValue()).build() + ), + nodeSettings + ); + assertTrue(indexSettings.getMergePolicy(false) instanceof OpenSearchTieredMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof OpenSearchTieredMergePolicy); + + // 3. INDEX_MERGE_POLICY set as log_byte_size + // assert both index and time-series-index merge policy is set as log_byte_size + indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.LOG_BYTE_SIZE.getValue()) + .build() + ); + assertTrue(indexSettings.getMergePolicy(false) instanceof LogByteSizeMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof LogByteSizeMergePolicy); + + // 3.1 node setting TIME_SERIES_INDEX_MERGE_POLICY is set as tiered + // assert both index and time-series-index merge policy is set as log_byte_size + nodeSettings = Settings.builder() + .put(IndexSettings.TIME_SERIES_INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.TIERED.getValue()) + .build(); + indexSettings = new IndexSettings( + newIndexMeta( + "test", + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.LOG_BYTE_SIZE.getValue()) + .build() + ), + nodeSettings + ); + assertTrue(indexSettings.getMergePolicy(false) instanceof LogByteSizeMergePolicy); + assertTrue(indexSettings.getMergePolicy(true) instanceof LogByteSizeMergePolicy); + + } + + public void testInvalidMergePolicy() throws IOException { + + final Settings invalidSettings = Settings.builder().put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "invalid").build(); + IllegalArgumentException exc1 = expectThrows( + IllegalArgumentException.class, + () -> IndexSettings.INDEX_MERGE_POLICY.get(invalidSettings) + ); + assertThat(exc1.getMessage(), containsString(" has unsupported policy specified: ")); + IllegalArgumentException exc2 = expectThrows( + IllegalArgumentException.class, + () -> indexSettings(invalidSettings).getMergePolicy(false) + ); + assertThat(exc2.getMessage(), containsString(" has unsupported policy specified: ")); + + final Settings invalidSettings2 = Settings.builder().put(IndexSettings.TIME_SERIES_INDEX_MERGE_POLICY.getKey(), "invalid").build(); + IllegalArgumentException exc3 = expectThrows( + IllegalArgumentException.class, + () -> IndexSettings.TIME_SERIES_INDEX_MERGE_POLICY.get(invalidSettings2) + ); + assertThat(exc3.getMessage(), containsString(" has unsupported policy specified: ")); + + IllegalArgumentException exc4 = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings(newIndexMeta("test", Settings.EMPTY), invalidSettings2).getMergePolicy(true) + ); + assertThat(exc4.getMessage(), containsString(" has unsupported policy specified: ")); + } + + public void testUpdateSettingsForLogByteSizeMergePolicy() throws IOException { + IndexSettings indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.LOG_BYTE_SIZE.getValue()) + .build() + ); + assertTrue(indexSettings.getMergePolicy(true) instanceof LogByteSizeMergePolicy); + assertThat(indexSettings.getMergePolicy(true).getNoCFSRatio(), equalTo(0.1)); + indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put(LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING.getKey(), 0.9) + .build() + ); + assertThat((indexSettings.getMergePolicy(true)).getNoCFSRatio(), equalTo(0.9)); + indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put(LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING.getKey(), 0.1) + .build() + ); + assertThat((indexSettings.getMergePolicy(true)).getNoCFSRatio(), equalTo(0.1)); + indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put(LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING.getKey(), 0.0) + .build() + ); + assertThat((indexSettings.getMergePolicy(true)).getNoCFSRatio(), equalTo(0.0)); + indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put(LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING.getKey(), "true") + .build() + ); + assertThat((indexSettings.getMergePolicy(true)).getNoCFSRatio(), equalTo(1.0)); + indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put(LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING.getKey(), "false") + .build() + ); + assertThat((indexSettings.getMergePolicy(true)).getNoCFSRatio(), equalTo(0.0)); } public void testTieredMergePolicySettingsUpdate() throws IOException { IndexSettings indexSettings = indexSettings(Settings.EMPTY); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), - MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getForceMergeDeletesPctAllowed(), + TieredMergePolicyProvider.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d ); @@ -102,21 +267,21 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { "index", Settings.builder() .put( - MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), - MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d + TieredMergePolicyProvider.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), + TieredMergePolicyProvider.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d ) .build() ) ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), - MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getForceMergeDeletesPctAllowed(), + TieredMergePolicyProvider.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), - MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getFloorSegmentMB(), + TieredMergePolicyProvider.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0 ); indexSettings.updateIndexMetadata( @@ -124,41 +289,41 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { "index", Settings.builder() .put( - MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), - new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB) + TieredMergePolicyProvider.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), + new ByteSizeValue(TieredMergePolicyProvider.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB) ) .build() ) ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), - new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getFloorSegmentMB(), + new ByteSizeValue(TieredMergePolicyProvider.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001 ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getMaxMergeAtOnce(), + TieredMergePolicyProvider.DEFAULT_MAX_MERGE_AT_ONCE ); indexSettings.updateIndexMetadata( newIndexMeta( "index", Settings.builder() .put( - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1 + TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), + TieredMergePolicyProvider.DEFAULT_MAX_MERGE_AT_ONCE - 1 ) .build() ) ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1 + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getMaxMergeAtOnce(), + TieredMergePolicyProvider.DEFAULT_MAX_MERGE_AT_ONCE - 1 ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), - MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getMaxMergedSegmentMB(), + TieredMergePolicyProvider.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001 ); indexSettings.updateIndexMetadata( @@ -166,21 +331,21 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { "index", Settings.builder() .put( - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), - new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1) + TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), + new ByteSizeValue(TieredMergePolicyProvider.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1) ) .build() ) ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), - new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getMaxMergedSegmentMB(), + new ByteSizeValue(TieredMergePolicyProvider.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001 ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), - MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getSegmentsPerTier(), + TieredMergePolicyProvider.DEFAULT_SEGMENTS_PER_TIER, 0 ); indexSettings.updateIndexMetadata( @@ -188,37 +353,37 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { "index", Settings.builder() .put( - MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), - MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1 + TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), + TieredMergePolicyProvider.DEFAULT_SEGMENTS_PER_TIER + 1 ) .build() ) ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), - MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getSegmentsPerTier(), + TieredMergePolicyProvider.DEFAULT_SEGMENTS_PER_TIER + 1, 0 ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(), - MergePolicyConfig.DEFAULT_DELETES_PCT_ALLOWED, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getDeletesPctAllowed(), + TieredMergePolicyProvider.DEFAULT_DELETES_PCT_ALLOWED, 0 ); indexSettings.updateIndexMetadata( newIndexMeta( "index", - Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 22).build() + Settings.builder().put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 22).build() ) ); - assertEquals(((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(), 22, 0); + assertEquals(((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getDeletesPctAllowed(), 22, 0); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, () -> indexSettings.updateIndexMetadata( newIndexMeta( "index", - Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 53).build() + Settings.builder().put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 53).build() ) ) ); @@ -226,50 +391,162 @@ public void testTieredMergePolicySettingsUpdate() throws IOException { assertThat(cause.getMessage(), containsString("must be <= 50.0")); indexSettings.updateIndexMetadata(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), - MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getForceMergeDeletesPctAllowed(), + TieredMergePolicyProvider.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), - new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getFloorSegmentMB(), + new ByteSizeValue(TieredMergePolicyProvider.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00 ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), - MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getMaxMergeAtOnce(), + TieredMergePolicyProvider.DEFAULT_MAX_MERGE_AT_ONCE ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), - new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getMaxMergedSegmentMB(), + new ByteSizeValue(TieredMergePolicyProvider.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001 ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), - MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getSegmentsPerTier(), + TieredMergePolicyProvider.DEFAULT_SEGMENTS_PER_TIER, 0 ); assertEquals( - ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(), - MergePolicyConfig.DEFAULT_DELETES_PCT_ALLOWED, + ((OpenSearchTieredMergePolicy) indexSettings.getMergePolicy(false)).getDeletesPctAllowed(), + TieredMergePolicyProvider.DEFAULT_DELETES_PCT_ALLOWED, 0 ); } + public void testLogByteSizeMergePolicySettingsUpdate() throws IOException { + + IndexSettings indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), IndexSettings.IndexMergePolicy.LOG_BYTE_SIZE.getValue()) + .build() + ); + assertEquals( + ((LogByteSizeMergePolicy) indexSettings.getMergePolicy(true)).getMergeFactor(), + LogByteSizeMergePolicyProvider.DEFAULT_MERGE_FACTOR + ); + + indexSettings.updateIndexMetadata( + newIndexMeta( + "index", + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put( + LogByteSizeMergePolicyProvider.INDEX_LBS_MERGE_POLICY_MERGE_FACTOR_SETTING.getKey(), + LogByteSizeMergePolicyProvider.DEFAULT_MERGE_FACTOR + 1 + ) + .build() + ) + ); + assertEquals( + ((LogByteSizeMergePolicy) indexSettings.getMergePolicy(true)).getMergeFactor(), + LogByteSizeMergePolicyProvider.DEFAULT_MERGE_FACTOR + 1 + ); + + indexSettings.updateIndexMetadata( + newIndexMeta( + "index", + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put( + LogByteSizeMergePolicyProvider.INDEX_LBS_MERGE_POLICY_MIN_MERGE_SETTING.getKey(), + new ByteSizeValue(LogByteSizeMergePolicyProvider.DEFAULT_MIN_MERGE.getMb() + 1, ByteSizeUnit.MB) + ) + .build() + ) + ); + + assertEquals( + ((LogByteSizeMergePolicy) indexSettings.getMergePolicy(true)).getMinMergeMB(), + new ByteSizeValue(LogByteSizeMergePolicyProvider.DEFAULT_MIN_MERGE.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), + 0.001 + ); + + indexSettings.updateIndexMetadata( + newIndexMeta( + "index", + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put( + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGE_SEGMENT_SETTING.getKey(), + new ByteSizeValue(LogByteSizeMergePolicyProvider.DEFAULT_MAX_MERGED_SEGMENT.getMb() + 100, ByteSizeUnit.MB) + ) + .build() + ) + ); + + assertEquals( + ((LogByteSizeMergePolicy) indexSettings.getMergePolicy(true)).getMaxMergeMB(), + new ByteSizeValue(TieredMergePolicyProvider.DEFAULT_MAX_MERGED_SEGMENT.getMb() + 100, ByteSizeUnit.MB).getMbFrac(), + 0.001 + ); + + indexSettings.updateIndexMetadata( + newIndexMeta( + "index", + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put( + LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGE_SEGMENT_FOR_FORCED_MERGE_SETTING.getKey(), + new ByteSizeValue( + LogByteSizeMergePolicyProvider.DEFAULT_MAX_MERGE_SEGMENT_FORCE_MERGE.getMb() - 100, + ByteSizeUnit.MB + ) + ) + .build() + ) + ); + assertEquals( + ((LogByteSizeMergePolicy) indexSettings.getMergePolicy(true)).getMaxMergeMBForForcedMerge(), + new ByteSizeValue(LogByteSizeMergePolicyProvider.DEFAULT_MAX_MERGE_SEGMENT_FORCE_MERGE.getMb() - 100, ByteSizeUnit.MB) + .getMbFrac(), + 0.001 + ); + + indexSettings.updateIndexMetadata( + newIndexMeta( + "index", + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put(LogByteSizeMergePolicyProvider.INDEX_LBS_MAX_MERGED_DOCS_SETTING.getKey(), 10000000) + .build() + ) + ); + assertEquals(((LogByteSizeMergePolicy) indexSettings.getMergePolicy(true)).getMaxMergeDocs(), 10000000); + + indexSettings.updateIndexMetadata( + newIndexMeta( + "index", + Settings.builder() + .put(IndexSettings.INDEX_MERGE_POLICY.getKey(), "log_byte_size") + .put(LogByteSizeMergePolicyProvider.INDEX_LBS_NO_CFS_RATIO_SETTING.getKey(), 0.1) + .build() + ) + ); + assertEquals(indexSettings.getMergePolicy(true).getNoCFSRatio(), 0.1, 0.0); + } + public Settings build(String value) { - return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); + return Settings.builder().put(TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); } public Settings build(double value) { - return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); + return Settings.builder().put(TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); } public Settings build(int value) { - return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); + return Settings.builder().put(TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); } public Settings build(boolean value) { - return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); + return Settings.builder().put(TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build(); } } diff --git a/server/src/test/java/org/opensearch/index/MergeSchedulerSettingsTests.java b/server/src/test/java/org/opensearch/index/MergeSchedulerSettingsTests.java index 2443ee1ab40be..baaf584702f78 100644 --- a/server/src/test/java/org/opensearch/index/MergeSchedulerSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/MergeSchedulerSettingsTests.java @@ -92,8 +92,8 @@ public void testUpdateAutoThrottleSettings() throws Exception { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1") .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "2") .put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "true"); @@ -123,8 +123,8 @@ public void testUpdateMergeMaxThreadCount() throws Exception { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") - .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2") + .put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2") .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "10000") .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "10000"); IndexSettings settings = new IndexSettings(newIndexMeta("index", builder.build()), Settings.EMPTY); diff --git a/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java index 9c8f9896850c6..c88c86d51be08 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandTests.java @@ -58,7 +58,7 @@ import org.opensearch.env.TestEnvironment; import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.index.IndexSettings; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineCreationFailureException; import org.opensearch.index.engine.InternalEngineFactory; @@ -134,7 +134,7 @@ public void setup() throws IOException { final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_INDEX_UUID, shardId.getIndex().getUUID()) .build(); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java index 11d916616578d..ad90255a3cc3f 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java @@ -54,7 +54,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexSettings; -import org.opensearch.index.MergePolicyConfig; +import org.opensearch.index.MergePolicyProvider; import org.opensearch.index.VersionType; import org.opensearch.index.engine.DocIdSeqNoAndSource; import org.opensearch.index.engine.Engine; @@ -168,7 +168,7 @@ public void testRecoveryWithOutOfOrderDeleteWithSoftDeletes() throws Exception { .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 10) // If soft-deletes is enabled, delete#1 will be reclaimed because its segment (segment_1) is fully deleted // index#0 will be retained if merge is disabled; otherwise it will be reclaimed because gcp=3 and retained_ops=0 - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(MergePolicyProvider.INDEX_MERGE_ENABLED, false) .build(); try (ReplicationGroup shards = createGroup(1, settings)) { shards.startAll(); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 6e064f943ca07..0b80c6e577f95 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -130,9 +130,9 @@ import org.opensearch.http.HttpInfo; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; -import org.opensearch.index.MergePolicyConfig; import org.opensearch.index.MergeSchedulerConfig; import org.opensearch.index.MockEngineFactoryPlugin; +import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.Segment; import org.opensearch.index.mapper.CompletionFieldMapper; @@ -500,7 +500,7 @@ protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builde private static Settings.Builder setRandomIndexMergeSettings(Random random, Settings.Builder builder) { if (random.nextBoolean()) { builder.put( - MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), + TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING.getKey(), (random.nextBoolean() ? random.nextDouble() : random.nextBoolean()).toString() ); } From d3bf230fe820f20121f3be35ca42b97cf804fea1 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Mon, 2 Oct 2023 13:47:48 -0700 Subject: [PATCH 085/388] Simplify initialization of Settings (#10194) There was a weird circular class-loading dependency between Settings and Settings.Builder that could result in ClassLoader deadlock if multiple threads trigger class-loading of Settings at the same time. Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + .../main/java/org/opensearch/common/settings/Settings.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7519e242c0acf..46f34abf72641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,6 +129,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045)) - Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082)) - Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089)) +- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194)) ### Security diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index 91e39e38f0379..d202feb0786bf 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -101,7 +101,7 @@ @PublicApi(since = "1.0.0") public final class Settings implements ToXContentFragment { - public static final Settings EMPTY = new Builder().build(); + public static final Settings EMPTY = new Settings(Collections.emptyMap(), null); /** The raw settings from the full key to raw string value. */ private final Map settings; @@ -757,7 +757,7 @@ public Set keySet() { @PublicApi(since = "1.0.0") public static class Builder { - public static final Settings EMPTY_SETTINGS = new Builder().build(); + public static final Settings EMPTY_SETTINGS = Settings.EMPTY; // we use a sorted map for consistent serialization when using getAsMap() private final Map map = new TreeMap<>(); From 9d0db5ef413ebc9f2eb7d8721ef74be273908094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Romain=20Tarti=C3=A8re?= Date: Mon, 2 Oct 2023 10:58:24 -1000 Subject: [PATCH 086/388] Remove spurious SGID (#9447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting the SGID bit on directories is maybe something some users will want to use, but setting it by default for all users does not really make sense and when packaging OpenSearch, we need to remove this customization when building packges. This was added to ElasticSearch to make it possible to manage the keystore as root while the service runs as an unprivileged user. Without the SGID trick, the generated keystore was owned by root and ElasticSearch could not access it. It is preferable to manage the keystore with non-root privileges, and this hack is not required in this case. Stick to the default permissions and remove this personalization. Signed-off-by: Romain Tartière --- CHANGELOG.md | 1 + distribution/packages/build.gradle | 8 ++++---- distribution/packages/src/deb/lintian/opensearch | 8 ++++---- .../test/java/org/opensearch/packaging/util/Packages.java | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f34abf72641..c8e4cc57593a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Deprecated ### Removed +- Remove spurious SGID bit on directories ([#9447](https://github.com/opensearch-project/OpenSearch/pull/9447)) ### Fixed - Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725)) diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 7914fcc172ef4..262ad6c802bbb 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -213,7 +213,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { configurationFile '/etc/opensearch/jvm.options' configurationFile '/etc/opensearch/log4j2.properties' from("${packagingFiles}") { - dirMode 02750 + dirMode 0750 into('/etc') permissionGroup 'opensearch' includeEmptyDirs true @@ -223,7 +223,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { } from("${packagingFiles}/etc/opensearch") { into('/etc/opensearch') - dirMode 02750 + dirMode 0750 fileMode 0660 permissionGroup 'opensearch' includeEmptyDirs true @@ -281,8 +281,8 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { dirMode mode } } - copyEmptyDir('/var/log/opensearch', 'opensearch', 'opensearch', 02750) - copyEmptyDir('/var/lib/opensearch', 'opensearch', 'opensearch', 02750) + copyEmptyDir('/var/log/opensearch', 'opensearch', 'opensearch', 0750) + copyEmptyDir('/var/lib/opensearch', 'opensearch', 'opensearch', 0750) copyEmptyDir('/usr/share/opensearch/plugins', 'root', 'root', 0755) into '/usr/share/opensearch' diff --git a/distribution/packages/src/deb/lintian/opensearch b/distribution/packages/src/deb/lintian/opensearch index 854b23131ecbc..e6db8e8c6b322 100644 --- a/distribution/packages/src/deb/lintian/opensearch +++ b/distribution/packages/src/deb/lintian/opensearch @@ -15,11 +15,11 @@ missing-dep-on-jarwrapper # we prefer to not make our config and log files world readable non-standard-file-perm etc/default/opensearch 0660 != 0644 -non-standard-dir-perm etc/opensearch/ 2750 != 0755 -non-standard-dir-perm etc/opensearch/jvm.options.d/ 2750 != 0755 +non-standard-dir-perm etc/opensearch/ 0750 != 0755 +non-standard-dir-perm etc/opensearch/jvm.options.d/ 0750 != 0755 non-standard-file-perm etc/opensearch/* -non-standard-dir-perm var/lib/opensearch/ 2750 != 0755 -non-standard-dir-perm var/log/opensearch/ 2750 != 0755 +non-standard-dir-perm var/lib/opensearch/ 0750 != 0755 +non-standard-dir-perm var/log/opensearch/ 0750 != 0755 executable-is-not-world-readable etc/init.d/opensearch 0750 non-standard-file-permissions-for-etc-init.d-script etc/init.d/opensearch 0750 != 0755 diff --git a/qa/os/src/test/java/org/opensearch/packaging/util/Packages.java b/qa/os/src/test/java/org/opensearch/packaging/util/Packages.java index b80ae422bda9a..e9ebf28042b46 100644 --- a/qa/os/src/test/java/org/opensearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/opensearch/packaging/util/Packages.java @@ -194,11 +194,11 @@ private static void verifyInstallation(Installation opensearch, Distribution dis // we shell out here because java's posix file permission view doesn't support special modes assertThat(opensearch.config, file(Directory, "root", "opensearch", p750)); - assertThat(sh.run("find \"" + opensearch.config + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750")); + assertThat(sh.run("find \"" + opensearch.config + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("750")); final Path jvmOptionsDirectory = opensearch.config.resolve("jvm.options.d"); assertThat(jvmOptionsDirectory, file(Directory, "root", "opensearch", p750)); - assertThat(sh.run("find \"" + jvmOptionsDirectory + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750")); + assertThat(sh.run("find \"" + jvmOptionsDirectory + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("750")); Stream.of("opensearch.keystore", "opensearch.yml", "jvm.options", "log4j2.properties") .forEach(configFile -> assertThat(opensearch.config(configFile), file(File, "root", "opensearch", p660))); From 1d66af346f7863bbca44a3102a1af2fed5fd3804 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 2 Oct 2023 17:51:47 -0400 Subject: [PATCH 087/388] Fix: register mulitple extensions. (#10256) * Fix: register mulitple extensions. Signed-off-by: dblock * Updated CHANGELOG. Signed-off-by: dblock * Added tests. Signed-off-by: dblock --------- Signed-off-by: dblock --- CHANGELOG.md | 1 + .../extensions/ExtensionsManager.java | 21 ++- .../rest/RestActionsRequestHandler.java | 3 + .../rest/RestInitializeExtensionAction.java | 3 +- .../rest/RestSendToExtensionAction.java | 2 +- .../extensions/ExtensionsManagerTests.java | 145 +++++++++++++++++- .../RestInitializeExtensionActionTests.java | 19 +-- .../rest/RestSendToExtensionActionTests.java | 6 +- 8 files changed, 176 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8e4cc57593a4..fb37533d4c834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,6 +131,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082)) - Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089)) - Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194)) +- Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256)) ### Security diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 9f9ba548143c6..b531abcb845d7 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -300,7 +300,7 @@ private void registerRequestHandler(DynamicActionRegistry dynamicActionRegistry) * Loads a single extension * @param extension The extension to be loaded */ - public void loadExtension(Extension extension) throws IOException { + public DiscoveryExtensionNode loadExtension(Extension extension) throws IOException { validateExtension(extension); DiscoveryExtensionNode discoveryExtensionNode = new DiscoveryExtensionNode( extension.getName(), @@ -314,6 +314,12 @@ public void loadExtension(Extension extension) throws IOException { extensionIdMap.put(extension.getUniqueId(), discoveryExtensionNode); extensionSettingsMap.put(extension.getUniqueId(), extension); logger.info("Loaded extension with uniqueId " + extension.getUniqueId() + ": " + extension); + return discoveryExtensionNode; + } + + public void initializeExtension(Extension extension) throws IOException { + DiscoveryExtensionNode node = loadExtension(extension); + initializeExtensionNode(node); } private void validateField(String fieldName, String value) throws IOException { @@ -340,11 +346,11 @@ private void validateExtension(Extension extension) throws IOException { */ public void initialize() { for (DiscoveryExtensionNode extension : extensionIdMap.values()) { - initializeExtension(extension); + initializeExtensionNode(extension); } } - private void initializeExtension(DiscoveryExtensionNode extension) { + public void initializeExtensionNode(DiscoveryExtensionNode extensionNode) { final CompletableFuture inProgressFuture = new CompletableFuture<>(); final TransportResponseHandler initializeExtensionResponseHandler = new TransportResponseHandler< @@ -384,7 +390,8 @@ public String executor() { transportService.getThreadPool().generic().execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { - extensionIdMap.remove(extension.getId()); + logger.warn("Error registering extension: " + extensionNode.getId(), e); + extensionIdMap.remove(extensionNode.getId()); if (e.getCause() instanceof ConnectTransportException) { logger.info("No response from extension to request.", e); throw (ConnectTransportException) e.getCause(); @@ -399,11 +406,11 @@ public void onFailure(Exception e) { @Override protected void doRun() throws Exception { - transportService.connectToExtensionNode(extension); + transportService.connectToExtensionNode(extensionNode); transportService.sendRequest( - extension, + extensionNode, REQUEST_EXTENSION_ACTION_NAME, - new InitializeExtensionRequest(transportService.getLocalNode(), extension, issueServiceAccount(extension)), + new InitializeExtensionRequest(transportService.getLocalNode(), extensionNode, issueServiceAccount(extensionNode)), initializeExtensionResponseHandler ); } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java b/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java index 97851cbd394a0..383796f0c3b44 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java @@ -62,6 +62,9 @@ public TransportResponse handleRegisterRestActionsRequest( DynamicActionRegistry dynamicActionRegistry ) throws Exception { DiscoveryExtensionNode discoveryExtensionNode = extensionIdMap.get(restActionsRequest.getUniqueId()); + if (discoveryExtensionNode == null) { + throw new IllegalStateException("Missing extension node for " + restActionsRequest.getUniqueId()); + } RestHandler handler = new RestSendToExtensionAction( restActionsRequest, discoveryExtensionNode, diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index 4b622b841a040..fc7c21a6eccd6 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -159,8 +159,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client extAdditionalSettings ); try { - extensionsManager.loadExtension(extension); - extensionsManager.initialize(); + extensionsManager.initializeExtension(extension); } catch (CompletionException e) { Throwable cause = e.getCause(); if (cause instanceof TimeoutException) { diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index 33f44a913dd8a..41783b89ccc69 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -150,7 +150,7 @@ public RestSendToExtensionAction( @Override public String getName() { - return SEND_TO_EXTENSION_ACTION; + return this.discoveryExtensionNode.getId() + ":" + SEND_TO_EXTENSION_ACTION; } @Override diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index f243a924f4e63..c61afdd5c5261 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -36,6 +36,7 @@ import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.discovery.InitializeExtensionRequest; import org.opensearch.env.Environment; import org.opensearch.env.EnvironmentSettingsResponse; import org.opensearch.extensions.ExtensionsSettings.Extension; @@ -77,6 +78,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -409,19 +411,94 @@ public void testInitialize() throws Exception { ) ); - // Test needs to be changed to mock the connection between the local node and an extension. Assert statment is commented out for - // now. + // Test needs to be changed to mock the connection between the local node and an extension. // Link to issue: https://github.com/opensearch-project/OpenSearch/issues/4045 // mockLogAppender.assertAllExpectationsMatched(); } } + public void testInitializeExtension() throws Exception { + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + + TransportService mockTransportService = spy( + new TransportService( + Settings.EMPTY, + mock(Transport.class), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + x -> null, + null, + Collections.emptySet(), + NoopTracer.INSTANCE + ) + ); + + doNothing().when(mockTransportService).connectToExtensionNode(any(DiscoveryExtensionNode.class)); + + doNothing().when(mockTransportService) + .sendRequest(any(DiscoveryExtensionNode.class), anyString(), any(InitializeExtensionRequest.class), any()); + + extensionsManager.initializeServicesAndRestHandler( + actionModule, + settingsModule, + mockTransportService, + clusterService, + settings, + client, + identityService + ); + + Extension firstExtension = new Extension( + "firstExtension", + "uniqueid1", + "127.0.0.0", + "9301", + "0.0.7", + "2.0.0", + "2.0.0", + List.of(), + null + ); + + extensionsManager.initializeExtension(firstExtension); + + Extension secondExtension = new Extension( + "secondExtension", + "uniqueid2", + "127.0.0.0", + "9301", + "0.0.7", + "2.0.0", + "2.0.0", + List.of(), + null + ); + + extensionsManager.initializeExtension(secondExtension); + + ThreadPool.terminate(threadPool, 3, TimeUnit.SECONDS); + + verify(mockTransportService, times(2)).connectToExtensionNode(any(DiscoveryExtensionNode.class)); + + verify(mockTransportService, times(2)).sendRequest( + any(DiscoveryExtensionNode.class), + anyString(), + any(InitializeExtensionRequest.class), + any() + ); + } + public void testHandleRegisterRestActionsRequest() throws Exception { ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; + + extensionsManager.loadExtension( + new Extension("firstExtension", uniqueIdStr, "127.0.0.0", "9300", "0.0.7", "3.0.0", "3.0.0", List.of(), null) + ); + List actionsList = List.of("GET /foo foo", "PUT /bar bar", "POST /baz baz"); List deprecatedActionsList = List.of("GET /deprecated/foo foo_deprecated", "It's deprecated!"); RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest(uniqueIdStr, actionsList, deprecatedActionsList); @@ -431,6 +508,58 @@ public void testHandleRegisterRestActionsRequest() throws Exception { assertTrue(((AcknowledgedResponse) response).getStatus()); } + public void testHandleRegisterRestActionsRequestRequiresDiscoveryNode() throws Exception { + + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + initialize(extensionsManager); + + RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest("uniqueId1", List.of(), List.of()); + + expectThrows( + IllegalStateException.class, + () -> extensionsManager.getRestActionsRequestHandler() + .handleRegisterRestActionsRequest(registerActionsRequest, actionModule.getDynamicActionRegistry()) + ); + } + + public void testHandleRegisterRestActionsRequestMultiple() throws Exception { + + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + initialize(extensionsManager); + + List actionsList = List.of("GET /foo foo", "PUT /bar bar", "POST /baz baz"); + List deprecatedActionsList = List.of("GET /deprecated/foo foo_deprecated", "It's deprecated!"); + for (int i = 0; i < 2; i++) { + String uniqueIdStr = "uniqueid-%d" + i; + + Set> additionalSettings = extAwarePlugin.getExtensionSettings().stream().collect(Collectors.toSet()); + ExtensionScopedSettings extensionScopedSettings = new ExtensionScopedSettings(additionalSettings); + Extension firstExtension = new Extension( + "Extension %s" + i, + uniqueIdStr, + "127.0.0.0", + "9300", + "0.0.7", + "3.0.0", + "3.0.0", + List.of(), + extensionScopedSettings + ); + + extensionsManager.loadExtension(firstExtension); + + RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest( + uniqueIdStr, + actionsList, + deprecatedActionsList + ); + TransportResponse response = extensionsManager.getRestActionsRequestHandler() + .handleRegisterRestActionsRequest(registerActionsRequest, actionModule.getDynamicActionRegistry()); + assertEquals(AcknowledgedResponse.class, response.getClass()); + assertTrue(((AcknowledgedResponse) response).getStatus()); + } + } + public void testHandleRegisterSettingsRequest() throws Exception { ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); @@ -452,6 +581,9 @@ public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Excep initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; + extensionsManager.loadExtension( + new Extension("firstExtension", uniqueIdStr, "127.0.0.0", "9300", "0.0.7", "3.0.0", "3.0.0", List.of(), null) + ); List actionsList = List.of("FOO /foo", "PUT /bar", "POST /baz"); List deprecatedActionsList = List.of("GET /deprecated/foo", "It's deprecated!"); RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest(uniqueIdStr, actionsList, deprecatedActionsList); @@ -467,6 +599,9 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() th initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; + extensionsManager.loadExtension( + new Extension("firstExtension", uniqueIdStr, "127.0.0.0", "9300", "0.0.7", "3.0.0", "3.0.0", List.of(), null) + ); List actionsList = List.of("GET /foo", "PUT /bar", "POST /baz"); List deprecatedActionsList = List.of("FOO /deprecated/foo", "It's deprecated!"); RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest(uniqueIdStr, actionsList, deprecatedActionsList); @@ -481,6 +616,9 @@ public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exceptio ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; + extensionsManager.loadExtension( + new Extension("firstExtension", uniqueIdStr, "127.0.0.0", "9300", "0.0.7", "3.0.0", "3.0.0", List.of(), null) + ); List actionsList = List.of("GET", "PUT /bar", "POST /baz"); List deprecatedActionsList = List.of("GET /deprecated/foo", "It's deprecated!"); RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest(uniqueIdStr, actionsList, deprecatedActionsList); @@ -495,6 +633,9 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throw ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; + extensionsManager.loadExtension( + new Extension("firstExtension", uniqueIdStr, "127.0.0.0", "9300", "0.0.7", "3.0.0", "3.0.0", List.of(), null) + ); List actionsList = List.of("GET /foo", "PUT /bar", "POST /baz"); List deprecatedActionsList = List.of("GET", "It's deprecated!"); RegisterRestActionsRequest registerActionsRequest = new RegisterRestActionsRequest(uniqueIdStr, actionsList, deprecatedActionsList); diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index cdddf8e9be1be..e237214ab88f5 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -19,8 +19,9 @@ import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.ExtensionsManager; -import org.opensearch.extensions.ExtensionsSettings; +import org.opensearch.extensions.ExtensionsSettings.Extension; import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestRequest; import org.opensearch.telemetry.tracing.noop.NoopTracer; @@ -160,8 +161,8 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettings() th // optionally, you can stub out some methods: when(spy.getAdditionalSettings()).thenCallRealMethod(); - Mockito.doCallRealMethod().when(spy).loadExtension(any(ExtensionsSettings.Extension.class)); - Mockito.doNothing().when(spy).initialize(); + Mockito.doCallRealMethod().when(spy).loadExtension(any(Extension.class)); + Mockito.doNothing().when(spy).initializeExtensionNode(any(DiscoveryExtensionNode.class)); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(spy); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" @@ -177,10 +178,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettings() th FakeRestChannel channel = new FakeRestChannel(request, false, 0); restInitializeExtensionAction.handleRequest(request, channel, null); - assertEquals(channel.capturedResponse().status(), RestStatus.ACCEPTED); + assertEquals(RestStatus.ACCEPTED, channel.capturedResponse().status()); assertTrue(channel.capturedResponse().content().utf8ToString().contains("A request to initialize an extension has been sent.")); - Optional extension = spy.lookupExtensionSettingsById("ad-extension"); + Optional extension = spy.lookupExtensionSettingsById("ad-extension"); assertTrue(extension.isPresent()); assertEquals(true, extension.get().getAdditionalSettings().get(boolSetting)); assertEquals("customSetting", extension.get().getAdditionalSettings().get(stringSetting)); @@ -210,8 +211,8 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsing // optionally, you can stub out some methods: when(spy.getAdditionalSettings()).thenCallRealMethod(); - Mockito.doCallRealMethod().when(spy).loadExtension(any(ExtensionsSettings.Extension.class)); - Mockito.doNothing().when(spy).initialize(); + Mockito.doCallRealMethod().when(spy).loadExtension(any(Extension.class)); + Mockito.doNothing().when(spy).initializeExtensionNode(any(DiscoveryExtensionNode.class)); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(spy); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" @@ -227,10 +228,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsing FakeRestChannel channel = new FakeRestChannel(request, false, 0); restInitializeExtensionAction.handleRequest(request, channel, null); - assertEquals(channel.capturedResponse().status(), RestStatus.ACCEPTED); + assertEquals(RestStatus.ACCEPTED, channel.capturedResponse().status()); assertTrue(channel.capturedResponse().content().utf8ToString().contains("A request to initialize an extension has been sent.")); - Optional extension = spy.lookupExtensionSettingsById("ad-extension"); + Optional extension = spy.lookupExtensionSettingsById("ad-extension"); assertTrue(extension.isPresent()); assertEquals(false, extension.get().getAdditionalSettings().get(boolSetting)); assertEquals("default", extension.get().getAdditionalSettings().get(stringSetting)); diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index ee36ea170e270..fe738ff7d85e6 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -150,7 +150,7 @@ public void testRestSendToExtensionAction() throws Exception { identityService ); - assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); + assertEquals("uniqueid1:send_to_extension_action", restSendToExtensionAction.getName()); List expected = new ArrayList<>(); String uriPrefix = "/_extensions/_uniqueid1"; expected.add(new Route(Method.GET, uriPrefix + "/foo")); @@ -183,7 +183,7 @@ public void testRestSendToExtensionActionWithNamedRoute() throws Exception { identityService ); - assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); + assertEquals("uniqueid1:send_to_extension_action", restSendToExtensionAction.getName()); List expected = new ArrayList<>(); String uriPrefix = "/_extensions/_uniqueid1"; NamedRoute nr1 = new NamedRoute.Builder().method(Method.GET).path(uriPrefix + "/foo").uniqueName("foo").build(); @@ -229,7 +229,7 @@ public void testRestSendToExtensionActionWithNamedRouteAndLegacyActionName() thr identityService ); - assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); + assertEquals("uniqueid1:send_to_extension_action", restSendToExtensionAction.getName()); List expected = new ArrayList<>(); String uriPrefix = "/_extensions/_uniqueid1"; NamedRoute nr1 = new NamedRoute.Builder().method(Method.GET) From beb25b1957321234aa3642977b0f7144318c8d8a Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 2 Oct 2023 18:21:58 -0400 Subject: [PATCH 088/388] Bump netty from 4.1.97.Final to 4.1.99.Final (#10306) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../licenses/netty-buffer-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.99.Final.jar.sha1 | 1 + .../transport-netty4/licenses/netty-codec-4.1.97.Final.jar.sha1 | 1 - .../transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.99.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.97.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-socks-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-socks-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-handler-proxy-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-handler-proxy-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.99.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.97.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.99.Final.jar.sha1 | 1 + .../repository-hdfs/licenses/netty-all-4.1.97.Final.jar.sha1 | 1 - .../repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-buffer-4.1.97.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-codec-4.1.97.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.99.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-common-4.1.97.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-handler-4.1.97.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-resolver-4.1.97.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.99.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.1.97.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.1.99.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.97.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.99.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-buffer-4.1.97.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-codec-4.1.97.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.99.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-common-4.1.97.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-handler-4.1.97.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-resolver-4.1.97.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.97.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.99.Final.jar.sha1 | 1 + 68 files changed, 35 insertions(+), 34 deletions(-) delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-common-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.97.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.97.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.97.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.97.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.97.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.97.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.97.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index fb37533d4c834..9b4e6408f3491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` - Bump Lucene from 9.7.0 to 9.8.0 ([#10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) - Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302)) +- Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 243a1b2c6f57e..e54a5a1089a93 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -28,7 +28,7 @@ jakarta_annotation = 1.3.5 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.13.0 -netty = 4.1.97.Final +netty = 4.1.99.Final joda = 2.12.2 # client dependencies diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.97.Final.jar.sha1 deleted file mode 100644 index 8430355365996..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f8f3d8644afa5e6e1a40a3a6aeb9d9aa970ecb4f \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..5b393be40e945 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9f02dcb9b15a647a56af210dffdc294a57922fb0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.97.Final.jar.sha1 deleted file mode 100644 index 7a36dc1f2724f..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -384ba4d75670befbedb45c4d3b497a93639c206d \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..45ea27d29a183 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9984cbd6e5d55c768f198e975d8aaf7fd42a4602 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.97.Final.jar.sha1 deleted file mode 100644 index 37b78a32f741f..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af78acec783ffd77c63d8aeecc21041fd39ac54f \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..6bb7fcd68b272 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +7142095066eaebd5f29b88c41af7b383b6a953f6 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 deleted file mode 100644 index cbf685a6d79d3..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -893888d09a7bef0d0ba973d7471943e765d0fd08 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..f9bdefc6dd965 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +c5a3481c4bb9732a3a94fb63cf916141a1a14669 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.97.Final.jar.sha1 deleted file mode 100644 index 1bdfec3aae6ba..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7cceacaf11df8dc63f23d0fb58e9d4640fc88404 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..d53adfa649f5f --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +278f6dfa49d6bd75c40ae1470eb165716f87dce0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.97.Final.jar.sha1 deleted file mode 100644 index 8b7b50a6fc9c6..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -abb86c6906bf512bf2b797a41cd7d2e8d3cd7c36 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..258f7c957dda0 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +742693761d7ea4c038bccfda96bb38194720b80d \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.97.Final.jar.sha1 deleted file mode 100644 index 032959e98d009..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cec8348108dc76c47cf87c669d514be52c922144 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..b8bc0a4370f58 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +080e45397d9d5b134477de3ffd0f94283b908621 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.97.Final.jar.sha1 deleted file mode 100644 index 107863c1b3c9d..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f37380d23c9bb079bc702910833b2fd532c9abd0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..247975e0a64c7 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9ca2e3ae19a6713b749df154622115f480b6716c \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 deleted file mode 100644 index f736d37d071b7..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d469d84265ab70095b01b40886cabdd433b6e664 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..6b7b66ea768e3 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +cb0fc6c31c387404212949c57950b5d72ce908b9 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.97.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.97.Final.jar.sha1 deleted file mode 100644 index f592ac8312a5d..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d266d079ef33cf93a16b382d64dd15d562df1159 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..6c1112ed49775 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +21c76a42a468faafac6c84f8aca775073fc8e345 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 deleted file mode 100644 index cbf685a6d79d3..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -893888d09a7bef0d0ba973d7471943e765d0fd08 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..f9bdefc6dd965 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +c5a3481c4bb9732a3a94fb63cf916141a1a14669 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.97.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.97.Final.jar.sha1 deleted file mode 100644 index d06147a0ba646..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -30e8fa29a349db5a933225d61891b8802836bb79 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..717703c36e1ab --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +259bf1c5178c3e23bb89a2fab59b6d22846e3fa6 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.97.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.97.Final.jar.sha1 deleted file mode 100644 index 67c3a763d26fa..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a99ecef0e1d86a92e40a7c89805c236d9cd7493e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..3f69ae54c5d4a --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +8c8a89ea89b06e120c57bdb3db14b9a47ca30bb3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.97.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.97.Final.jar.sha1 deleted file mode 100644 index 60fd706436ae7..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2c50f835777ecd4535e15b552b5d9ccb26a2504f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..adef44a4e7da7 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +804d8b752847923d3bb81f24de604597047c9b2e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 deleted file mode 100644 index f736d37d071b7..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d469d84265ab70095b01b40886cabdd433b6e664 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..6b7b66ea768e3 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +cb0fc6c31c387404212949c57950b5d72ce908b9 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.97.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.97.Final.jar.sha1 deleted file mode 100644 index c6fa4cc175222..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -afec3c414a0ab7264a66a7572e9e9d3a19a3e0e5 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..0756635018837 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +a45aa70bc50d0500da5cdcd595cc838d87ada987 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.97.Final.jar.sha1 deleted file mode 100644 index 8430355365996..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f8f3d8644afa5e6e1a40a3a6aeb9d9aa970ecb4f \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..5b393be40e945 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9f02dcb9b15a647a56af210dffdc294a57922fb0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.97.Final.jar.sha1 deleted file mode 100644 index 7a36dc1f2724f..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -384ba4d75670befbedb45c4d3b497a93639c206d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..45ea27d29a183 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9984cbd6e5d55c768f198e975d8aaf7fd42a4602 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.97.Final.jar.sha1 deleted file mode 100644 index 37b78a32f741f..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af78acec783ffd77c63d8aeecc21041fd39ac54f \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..6bb7fcd68b272 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +7142095066eaebd5f29b88c41af7b383b6a953f6 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 deleted file mode 100644 index cbf685a6d79d3..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -893888d09a7bef0d0ba973d7471943e765d0fd08 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..f9bdefc6dd965 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +c5a3481c4bb9732a3a94fb63cf916141a1a14669 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.97.Final.jar.sha1 deleted file mode 100644 index 1bdfec3aae6ba..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7cceacaf11df8dc63f23d0fb58e9d4640fc88404 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..d53adfa649f5f --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +278f6dfa49d6bd75c40ae1470eb165716f87dce0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.97.Final.jar.sha1 deleted file mode 100644 index 8b7b50a6fc9c6..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -abb86c6906bf512bf2b797a41cd7d2e8d3cd7c36 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..258f7c957dda0 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +742693761d7ea4c038bccfda96bb38194720b80d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.97.Final.jar.sha1 deleted file mode 100644 index 032959e98d009..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cec8348108dc76c47cf87c669d514be52c922144 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..b8bc0a4370f58 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +080e45397d9d5b134477de3ffd0f94283b908621 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.97.Final.jar.sha1 deleted file mode 100644 index 107863c1b3c9d..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f37380d23c9bb079bc702910833b2fd532c9abd0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..247975e0a64c7 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9ca2e3ae19a6713b749df154622115f480b6716c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.97.Final.jar.sha1 deleted file mode 100644 index 8e40c8826d76d..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -795da37ded759e862457a82d9d92c4d39ce8ecee \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..75b64ad4197d8 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +961bd5b8d97ea6a07168176462f398089a24b5c8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 deleted file mode 100644 index f736d37d071b7..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d469d84265ab70095b01b40886cabdd433b6e664 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..6b7b66ea768e3 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +cb0fc6c31c387404212949c57950b5d72ce908b9 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.97.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.97.Final.jar.sha1 deleted file mode 100644 index 8430355365996..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f8f3d8644afa5e6e1a40a3a6aeb9d9aa970ecb4f \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..5b393be40e945 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9f02dcb9b15a647a56af210dffdc294a57922fb0 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.97.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.97.Final.jar.sha1 deleted file mode 100644 index 7a36dc1f2724f..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -384ba4d75670befbedb45c4d3b497a93639c206d \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..45ea27d29a183 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9984cbd6e5d55c768f198e975d8aaf7fd42a4602 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.97.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.97.Final.jar.sha1 deleted file mode 100644 index 37b78a32f741f..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af78acec783ffd77c63d8aeecc21041fd39ac54f \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..6bb7fcd68b272 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +7142095066eaebd5f29b88c41af7b383b6a953f6 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.97.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.97.Final.jar.sha1 deleted file mode 100644 index 1bdfec3aae6ba..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7cceacaf11df8dc63f23d0fb58e9d4640fc88404 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..d53adfa649f5f --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +278f6dfa49d6bd75c40ae1470eb165716f87dce0 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.97.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.97.Final.jar.sha1 deleted file mode 100644 index 8b7b50a6fc9c6..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -abb86c6906bf512bf2b797a41cd7d2e8d3cd7c36 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..258f7c957dda0 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +742693761d7ea4c038bccfda96bb38194720b80d \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.97.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.97.Final.jar.sha1 deleted file mode 100644 index 032959e98d009..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cec8348108dc76c47cf87c669d514be52c922144 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..b8bc0a4370f58 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +080e45397d9d5b134477de3ffd0f94283b908621 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.97.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.97.Final.jar.sha1 deleted file mode 100644 index 107863c1b3c9d..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.97.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f37380d23c9bb079bc702910833b2fd532c9abd0 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1 new file mode 100644 index 0000000000000..247975e0a64c7 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1 @@ -0,0 +1 @@ +9ca2e3ae19a6713b749df154622115f480b6716c \ No newline at end of file From 3a790c150e6a8b45d385341e276da6f226489515 Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Mon, 2 Oct 2023 22:37:17 -0700 Subject: [PATCH 089/388] Disable concurrent search when terminate_after is used (#10200) Signed-off-by: Jay Deng --- CHANGELOG.md | 1 + .../simple/ParameterizedSimpleSearchIT.java | 608 ----------------- .../search/simple/SimpleSearchIT.java | 645 +++++++++++++++++- .../search/DefaultSearchContext.java | 2 + 4 files changed, 645 insertions(+), 611 deletions(-) delete mode 100644 server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b4e6408f3491..9963eaef31d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042)) - [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122)) - Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246)) +- Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java deleted file mode 100644 index 719b75079da92..0000000000000 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/ParameterizedSimpleSearchIT.java +++ /dev/null @@ -1,608 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.search.simple; - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.opensearch.action.index.IndexRequestBuilder; -import org.opensearch.action.search.SearchPhaseExecutionException; -import org.opensearch.action.search.SearchRequestBuilder; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.WriteRequest.RefreshPolicy; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.search.rescore.QueryRescorerBuilder; -import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.ExecutionException; - -import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.opensearch.index.query.QueryBuilders.boolQuery; -import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.index.query.QueryBuilders.queryStringQuery; -import static org.opensearch.index.query.QueryBuilders.rangeQuery; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class ParameterizedSimpleSearchIT extends ParameterizedOpenSearchIntegTestCase { - - public ParameterizedSimpleSearchIT(Settings settings) { - super(settings); - } - - @ParametersFactory - public static Collection parameters() { - return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } - ); - } - - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); - } - - public void testSearchNullIndex() { - expectThrows( - NullPointerException.class, - () -> client().prepareSearch((String) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() - ); - - expectThrows( - NullPointerException.class, - () -> client().prepareSearch((String[]) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() - ); - - } - - public void testSearchRandomPreference() throws InterruptedException, ExecutionException { - createIndex("test"); - indexRandom( - true, - client().prepareIndex("test").setId("1").setSource("field", "value"), - client().prepareIndex("test").setId("2").setSource("field", "value"), - client().prepareIndex("test").setId("3").setSource("field", "value"), - client().prepareIndex("test").setId("4").setSource("field", "value"), - client().prepareIndex("test").setId("5").setSource("field", "value"), - client().prepareIndex("test").setId("6").setSource("field", "value") - ); - - int iters = scaledRandomIntBetween(10, 20); - for (int i = 0; i < iters; i++) { - String randomPreference = randomUnicodeOfLengthBetween(0, 4); - // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) - while (randomPreference.startsWith("_")) { - randomPreference = randomUnicodeOfLengthBetween(0, 4); - } - // id is not indexed, but lets see that we automatically convert to - SearchResponse searchResponse = client().prepareSearch() - .setQuery(QueryBuilders.matchAllQuery()) - .setPreference(randomPreference) - .get(); - assertHitCount(searchResponse, 6L); - - } - } - - public void testSimpleIp() throws Exception { - createIndex("test"); - - client().admin() - .indices() - .preparePutMapping("test") - .setSource( - XContentFactory.jsonBuilder() - .startObject() - .startObject(MapperService.SINGLE_MAPPING_NAME) - .startObject("properties") - .startObject("from") - .field("type", "ip") - .endObject() - .startObject("to") - .field("type", "ip") - .endObject() - .endObject() - .endObject() - .endObject() - ) - .get(); - - client().prepareIndex("test").setId("1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefreshPolicy(IMMEDIATE).get(); - - SearchResponse search = client().prepareSearch() - .setQuery(boolQuery().must(rangeQuery("from").lte("192.168.0.7")).must(rangeQuery("to").gte("192.168.0.7"))) - .get(); - - assertHitCount(search, 1L); - } - - public void testIpCidr() throws Exception { - createIndex("test"); - - client().admin() - .indices() - .preparePutMapping("test") - .setSource( - XContentFactory.jsonBuilder() - .startObject() - .startObject(MapperService.SINGLE_MAPPING_NAME) - .startObject("properties") - .startObject("ip") - .field("type", "ip") - .endObject() - .endObject() - .endObject() - .endObject() - ) - .get(); - ensureGreen(); - - client().prepareIndex("test").setId("1").setSource("ip", "192.168.0.1").get(); - client().prepareIndex("test").setId("2").setSource("ip", "192.168.0.2").get(); - client().prepareIndex("test").setId("3").setSource("ip", "192.168.0.3").get(); - client().prepareIndex("test").setId("4").setSource("ip", "192.168.1.4").get(); - client().prepareIndex("test").setId("5").setSource("ip", "2001:db8::ff00:42:8329").get(); - refresh(); - - SearchResponse search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(queryStringQuery("ip: 192.168.0.1")).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/32"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.0/24"))).get(); - assertHitCount(search, 3L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))).get(); - assertHitCount(search, 4L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))).get(); - assertHitCount(search, 4L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::ff00:42:8329/128"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::/64"))).get(); - assertHitCount(search, 1L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "::/0"))).get(); - assertHitCount(search, 5L); - - search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.5/32"))).get(); - assertHitCount(search, 0L); - - assertFailures( - client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0/0/0/0/0"))), - RestStatus.BAD_REQUEST, - containsString("Expected [ip/prefix] but was [0/0/0/0/0]") - ); - } - - public void testSimpleId() { - createIndex("test"); - - client().prepareIndex("test").setId("XXX1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); - // id is not indexed, but lets see that we automatically convert to - SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")).get(); - assertHitCount(searchResponse, 1L); - - searchResponse = client().prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")).get(); - assertHitCount(searchResponse, 1L); - } - - public void testSimpleDateRange() throws Exception { - createIndex("test"); - client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); - client().prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); - ensureGreen(); - refresh(); - SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 2L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lte("2010-01-06T02:00")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 2L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lt("2010-01-06T02:00")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 1L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00").lt("2010-01-06T02:00")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 0L); - - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")) - .get(); - assertHitCount(searchResponse, 2L); - } - - // TODO: combine this test with SimpleSearchIT.testSimpleTerminateAfterCount after - // https://github.com/opensearch-project/OpenSearch/issues/8371 - public void testSimpleTerminateAfterCountWithSizeAndTrackHits() throws Exception { - prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); - ensureGreen(); - int numDocs = randomIntBetween(15, 29); - List docbuilders = new ArrayList<>(numDocs); - - for (int i = 1; i <= numDocs; i++) { - String id = String.valueOf(i); - docbuilders.add(client().prepareIndex("test").setId(id).setSource("field", i)); - } - - indexRandom(true, docbuilders); - ensureGreen(); - refresh(); - - SearchResponse searchResponse; - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.matchAllQuery()) - .setTerminateAfter(numDocs) - .setSize(0) - .setTrackTotalHits(true) - .get(); - assertEquals(0, searchResponse.getFailedShards()); - } - - public void testSimpleIndexSortEarlyTerminate() throws Exception { - prepareCreate("test").setSettings( - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).put("index.sort.field", "rank") - ).setMapping("rank", "type=integer").get(); - ensureGreen(); - int max = randomIntBetween(3, 29); - List docbuilders = new ArrayList<>(max); - - for (int i = max - 1; i >= 0; i--) { - String id = String.valueOf(i); - docbuilders.add(client().prepareIndex("test").setId(id).setSource("rank", i)); - } - - indexRandom(true, docbuilders); - ensureGreen(); - refresh(); - - SearchResponse searchResponse; - for (int i = 1; i < max; i++) { - searchResponse = client().prepareSearch("test") - .addDocValueField("rank") - .setTrackTotalHits(false) - .addSort("rank", SortOrder.ASC) - .setSize(i) - .get(); - assertNull(searchResponse.getHits().getTotalHits()); - for (int j = 0; j < i; j++) { - assertThat(searchResponse.getHits().getAt(j).field("rank").getValue(), equalTo((long) j)); - } - } - } - - public void testInsaneFromAndSize() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE)); - assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE)); - } - - public void testTooLargeFromAndSize() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))); - assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1)); - assertWindowFails( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - ); - } - - public void testLargeFromAndSizeSucceeds() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2 - 1) - .get(), - 1 - ); - } - - public void testTooLargeFromAndSizeOkBySetting() throws Exception { - prepareCreate("idx").setSettings( - Settings.builder() - .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2) - ).get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .get(), - 1 - ); - } - - public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { - createIndex("idx"); - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings("idx") - .setSettings( - Settings.builder() - .put( - IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), - IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2 - ) - ) - .get() - ); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .get(), - 1 - ); - } - - public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception { - prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)).get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); - assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); - assertHitCount( - client().prepareSearch("idx") - .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) - .get(), - 1 - ); - } - - public void testTooLargeRescoreWindow() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertRescoreWindowFails(Integer.MAX_VALUE); - assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1); - } - - public void testTooLargeRescoreOkBySetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) - .get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testTooLargeRescoreOkByResultWindowSetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - prepareCreate("idx").setSettings( - Settings.builder() - .put( - IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window. - defaultMaxWindow * 2 - ) - ).get(); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testTooLargeRescoreOkByDynamicSetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - createIndex("idx"); - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings("idx") - .setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) - .get() - ); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception { - int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); - createIndex("idx"); - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings("idx") - .setSettings( - // Note that this is the RESULT window - Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2) - ) - .get() - ); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - assertHitCount( - client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), - 1 - ); - } - - public void testQueryNumericFieldWithRegex() throws Exception { - assertAcked(prepareCreate("idx").setMapping("num", "type=integer")); - ensureGreen("idx"); - - try { - client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get(); - fail("SearchPhaseExecutionException should have been thrown"); - } catch (SearchPhaseExecutionException ex) { - assertThat(ex.getRootCause().getMessage(), containsString("Can only use regexp queries on keyword and text fields")); - } - } - - public void testTermQueryBigInt() throws Exception { - prepareCreate("idx").setMapping("field", "type=keyword").get(); - ensureGreen("idx"); - - client().prepareIndex("idx") - .setId("1") - .setSource("{\"field\" : 80315953321748200608 }", MediaTypeRegistry.JSON) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .get(); - - String queryJson = "{ \"field\" : { \"value\" : 80315953321748200608 } }"; - XContentParser parser = createParser(JsonXContent.jsonXContent, queryJson); - parser.nextToken(); - TermQueryBuilder query = TermQueryBuilder.fromXContent(parser); - SearchResponse searchResponse = client().prepareSearch("idx").setQuery(query).get(); - assertEquals(1, searchResponse.getHits().getTotalHits().value); - } - - public void testTooLongRegexInRegexpQuery() throws Exception { - createIndex("idx"); - indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); - - int defaultMaxRegexLength = IndexSettings.MAX_REGEX_LENGTH_SETTING.get(Settings.EMPTY); - StringBuilder regexp = new StringBuilder(defaultMaxRegexLength); - while (regexp.length() <= defaultMaxRegexLength) { - regexp.append("]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*))*(?:,@(?:(?:\\r\\n)?[ \\t])*(?:[^()<>@,;:\\\\\".\\"); - } - SearchPhaseExecutionException e = expectThrows( - SearchPhaseExecutionException.class, - () -> client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", regexp.toString())).get() - ); - assertThat( - e.getRootCause().getMessage(), - containsString( - "The length of regex [" - + regexp.length() - + "] used in the Regexp Query request has exceeded " - + "the allowed maximum of [" - + defaultMaxRegexLength - + "]. " - + "This maximum can be set by changing the [" - + IndexSettings.MAX_REGEX_LENGTH_SETTING.getKey() - + "] index level setting." - ) - ); - } - - private void assertWindowFails(SearchRequestBuilder search) { - SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); - assertThat( - e.toString(), - containsString( - "Result window is too large, from + size must be less than or equal to: [" - + IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - ) - ); - assertThat(e.toString(), containsString("See the scroll api for a more efficient way to request large data sets")); - } - - private void assertRescoreWindowFails(int windowSize) { - SearchRequestBuilder search = client().prepareSearch("idx") - .addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(windowSize)); - SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); - assertThat( - e.toString(), - containsString( - "Rescore window [" - + windowSize - + "] is too large. It must " - + "be less than [" - + IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) - ) - ); - assertThat( - e.toString(), - containsString( - "This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting." - ) - ); - } -} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index 67e460653245e..95b36311f6b8b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -6,24 +6,283 @@ * compatible open source license. */ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + package org.opensearch.search.simple; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchPhaseExecutionException; +import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.rescore.QueryRescorerBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.test.ParameterizedOpenSearchIntegTestCase; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.concurrent.ExecutionException; +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.index.query.QueryBuilders.boolQuery; +import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.index.query.QueryBuilders.queryStringQuery; +import static org.opensearch.index.query.QueryBuilders.rangeQuery; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertFailures; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.apache.lucene.search.TotalHits.Relation.EQUAL_TO; +import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO; + +public class SimpleSearchIT extends ParameterizedOpenSearchIntegTestCase { + + public SimpleSearchIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() }, + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() } + ); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); + } + + public void testSearchNullIndex() { + expectThrows( + NullPointerException.class, + () -> client().prepareSearch((String) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() + ); + + expectThrows( + NullPointerException.class, + () -> client().prepareSearch((String[]) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get() + ); + + } + + public void testSearchRandomPreference() throws InterruptedException, ExecutionException { + createIndex("test"); + indexRandom( + true, + client().prepareIndex("test").setId("1").setSource("field", "value"), + client().prepareIndex("test").setId("2").setSource("field", "value"), + client().prepareIndex("test").setId("3").setSource("field", "value"), + client().prepareIndex("test").setId("4").setSource("field", "value"), + client().prepareIndex("test").setId("5").setSource("field", "value"), + client().prepareIndex("test").setId("6").setSource("field", "value") + ); + + int iters = scaledRandomIntBetween(10, 20); + for (int i = 0; i < iters; i++) { + String randomPreference = randomUnicodeOfLengthBetween(0, 4); + // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) + while (randomPreference.startsWith("_")) { + randomPreference = randomUnicodeOfLengthBetween(0, 4); + } + // id is not indexed, but lets see that we automatically convert to + SearchResponse searchResponse = client().prepareSearch() + .setQuery(QueryBuilders.matchAllQuery()) + .setPreference(randomPreference) + .get(); + assertHitCount(searchResponse, 6L); + + } + } + + public void testSimpleIp() throws Exception { + createIndex("test"); + + client().admin() + .indices() + .preparePutMapping("test") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject(MapperService.SINGLE_MAPPING_NAME) + .startObject("properties") + .startObject("from") + .field("type", "ip") + .endObject() + .startObject("to") + .field("type", "ip") + .endObject() + .endObject() + .endObject() + .endObject() + ) + .get(); + + client().prepareIndex("test").setId("1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefreshPolicy(IMMEDIATE).get(); + + SearchResponse search = client().prepareSearch() + .setQuery(boolQuery().must(rangeQuery("from").lte("192.168.0.7")).must(rangeQuery("to").gte("192.168.0.7"))) + .get(); + + assertHitCount(search, 1L); + } + + public void testIpCidr() throws Exception { + createIndex("test"); + + client().admin() + .indices() + .preparePutMapping("test") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject(MapperService.SINGLE_MAPPING_NAME) + .startObject("properties") + .startObject("ip") + .field("type", "ip") + .endObject() + .endObject() + .endObject() + .endObject() + ) + .get(); + ensureGreen(); + + client().prepareIndex("test").setId("1").setSource("ip", "192.168.0.1").get(); + client().prepareIndex("test").setId("2").setSource("ip", "192.168.0.2").get(); + client().prepareIndex("test").setId("3").setSource("ip", "192.168.0.3").get(); + client().prepareIndex("test").setId("4").setSource("ip", "192.168.1.4").get(); + client().prepareIndex("test").setId("5").setSource("ip", "2001:db8::ff00:42:8329").get(); + refresh(); + + SearchResponse search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(queryStringQuery("ip: 192.168.0.1")).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/32"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.0/24"))).get(); + assertHitCount(search, 3L); -public class SimpleSearchIT extends OpenSearchIntegTestCase { + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))).get(); + assertHitCount(search, 4L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))).get(); + assertHitCount(search, 4L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::ff00:42:8329/128"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::/64"))).get(); + assertHitCount(search, 1L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "::/0"))).get(); + assertHitCount(search, 5L); + + search = client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.5/32"))).get(); + assertHitCount(search, 0L); + + assertFailures( + client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0/0/0/0/0"))), + RestStatus.BAD_REQUEST, + containsString("Expected [ip/prefix] but was [0/0/0/0/0]") + ); + } + + public void testSimpleId() { + createIndex("test"); + + client().prepareIndex("test").setId("XXX1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + // id is not indexed, but lets see that we automatically convert to + SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")).get(); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")).get(); + assertHitCount(searchResponse, 1L); + } + + public void testSimpleDateRange() throws Exception { + createIndex("test"); + client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); + client().prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); + ensureGreen(); + refresh(); + SearchResponse searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lte("2010-01-06T02:00")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00").lt("2010-01-06T02:00")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00").lt("2010-01-06T02:00")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 0L); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")) + .get(); + assertHitCount(searchResponse, 2L); + } - // TODO: Move this test to ParameterizedSimpleSearchIT after https://github.com/opensearch-project/OpenSearch/issues/8371 public void testSimpleTerminateAfterCount() throws Exception { prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); ensureGreen(); @@ -40,13 +299,18 @@ public void testSimpleTerminateAfterCount() throws Exception { refresh(); SearchResponse searchResponse; + int size; for (int i = 1; i < max; i++) { + size = randomIntBetween(0, max); searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max)) .setTerminateAfter(i) + .setSize(size) + .setTrackTotalHits(true) .get(); assertHitCount(searchResponse, i); assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(Math.min(i, size), searchResponse.getHits().getHits().length); } searchResponse = client().prepareSearch("test") @@ -57,4 +321,379 @@ public void testSimpleTerminateAfterCount() throws Exception { assertHitCount(searchResponse, max); assertFalse(searchResponse.isTerminatedEarly()); } + + public void testSimpleTerminateAfterTrackTotalHitsUpTo() throws Exception { + prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); + ensureGreen(); + int numDocs = 29; + List docbuilders = new ArrayList<>(numDocs); + + for (int i = 1; i <= numDocs; i++) { + String id = String.valueOf(i); + docbuilders.add(client().prepareIndex("test").setId(id).setSource("field", i)); + } + + indexRandom(true, docbuilders); + ensureGreen(); + refresh(); + + // size=0 is a special case where topDocsCollector is not added + int size = randomIntBetween(0, 1); + SearchResponse searchResponse; + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(10) + .setSize(size) + .setTrackTotalHitsUpTo(5) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(GREATER_THAN_OR_EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(5) + .setSize(size) + .setTrackTotalHitsUpTo(10) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(5) + .setSize(size) + .setTrackTotalHitsUpTo(5) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(5) + .setSize(size) + .setTrackTotalHits(true) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(numDocs * 2) + .setSize(size) + .setTrackTotalHits(true) + .get(); + assertFalse(searchResponse.isTerminatedEarly()); + assertEquals(numDocs, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setSize(size) + .setTrackTotalHitsUpTo(5) + .get(); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(GREATER_THAN_OR_EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + } + + public void testSimpleIndexSortEarlyTerminate() throws Exception { + prepareCreate("test").setSettings( + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).put("index.sort.field", "rank") + ).setMapping("rank", "type=integer").get(); + ensureGreen(); + int max = randomIntBetween(3, 29); + List docbuilders = new ArrayList<>(max); + + for (int i = max - 1; i >= 0; i--) { + String id = String.valueOf(i); + docbuilders.add(client().prepareIndex("test").setId(id).setSource("rank", i)); + } + + indexRandom(true, docbuilders); + ensureGreen(); + refresh(); + + SearchResponse searchResponse; + for (int i = 1; i < max; i++) { + searchResponse = client().prepareSearch("test") + .addDocValueField("rank") + .setTrackTotalHits(false) + .addSort("rank", SortOrder.ASC) + .setSize(i) + .get(); + assertNull(searchResponse.getHits().getTotalHits()); + for (int j = 0; j < i; j++) { + assertThat(searchResponse.getHits().getAt(j).field("rank").getValue(), equalTo((long) j)); + } + } + } + + public void testInsaneFromAndSize() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE)); + assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE)); + } + + public void testTooLargeFromAndSize() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))); + assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1)); + assertWindowFails( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + ); + } + + public void testLargeFromAndSizeSucceeds() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2 - 1) + .get(), + 1 + ); + } + + public void testTooLargeFromAndSizeOkBySetting() throws Exception { + prepareCreate("idx").setSettings( + Settings.builder() + .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2) + ).get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .get(), + 1 + ); + } + + public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { + createIndex("idx"); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("idx") + .setSettings( + Settings.builder() + .put( + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), + IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2 + ) + ) + .get() + ); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) + .get(), + 1 + ); + } + + public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception { + prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)).get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); + assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1); + assertHitCount( + client().prepareSearch("idx") + .setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) + .get(), + 1 + ); + } + + public void testTooLargeRescoreWindow() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertRescoreWindowFails(Integer.MAX_VALUE); + assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1); + } + + public void testTooLargeRescoreOkBySetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) + .get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testTooLargeRescoreOkByResultWindowSetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + prepareCreate("idx").setSettings( + Settings.builder() + .put( + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window. + defaultMaxWindow * 2 + ) + ).get(); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testTooLargeRescoreOkByDynamicSetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + createIndex("idx"); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("idx") + .setSettings(Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2)) + .get() + ); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testTooLargeRescoreOkByDynamicResultWindowSetting() throws Exception { + int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY); + createIndex("idx"); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("idx") + .setSettings( + // Note that this is the RESULT window + Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2) + ) + .get() + ); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + assertHitCount( + client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(), + 1 + ); + } + + public void testQueryNumericFieldWithRegex() throws Exception { + assertAcked(prepareCreate("idx").setMapping("num", "type=integer")); + ensureGreen("idx"); + + try { + client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get(); + fail("SearchPhaseExecutionException should have been thrown"); + } catch (SearchPhaseExecutionException ex) { + assertThat(ex.getRootCause().getMessage(), containsString("Can only use regexp queries on keyword and text fields")); + } + } + + public void testTermQueryBigInt() throws Exception { + prepareCreate("idx").setMapping("field", "type=keyword").get(); + ensureGreen("idx"); + + client().prepareIndex("idx") + .setId("1") + .setSource("{\"field\" : 80315953321748200608 }", MediaTypeRegistry.JSON) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); + + String queryJson = "{ \"field\" : { \"value\" : 80315953321748200608 } }"; + XContentParser parser = createParser(JsonXContent.jsonXContent, queryJson); + parser.nextToken(); + TermQueryBuilder query = TermQueryBuilder.fromXContent(parser); + SearchResponse searchResponse = client().prepareSearch("idx").setQuery(query).get(); + assertEquals(1, searchResponse.getHits().getTotalHits().value); + } + + public void testTooLongRegexInRegexpQuery() throws Exception { + createIndex("idx"); + indexRandom(true, client().prepareIndex("idx").setSource("{}", MediaTypeRegistry.JSON)); + + int defaultMaxRegexLength = IndexSettings.MAX_REGEX_LENGTH_SETTING.get(Settings.EMPTY); + StringBuilder regexp = new StringBuilder(defaultMaxRegexLength); + while (regexp.length() <= defaultMaxRegexLength) { + regexp.append("]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*))*(?:,@(?:(?:\\r\\n)?[ \\t])*(?:[^()<>@,;:\\\\\".\\"); + } + SearchPhaseExecutionException e = expectThrows( + SearchPhaseExecutionException.class, + () -> client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", regexp.toString())).get() + ); + assertThat( + e.getRootCause().getMessage(), + containsString( + "The length of regex [" + + regexp.length() + + "] used in the Regexp Query request has exceeded " + + "the allowed maximum of [" + + defaultMaxRegexLength + + "]. " + + "This maximum can be set by changing the [" + + IndexSettings.MAX_REGEX_LENGTH_SETTING.getKey() + + "] index level setting." + ) + ); + } + + private void assertWindowFails(SearchRequestBuilder search) { + SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); + assertThat( + e.toString(), + containsString( + "Result window is too large, from + size must be less than or equal to: [" + + IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + ) + ); + assertThat(e.toString(), containsString("See the scroll api for a more efficient way to request large data sets")); + } + + private void assertRescoreWindowFails(int windowSize) { + SearchRequestBuilder search = client().prepareSearch("idx") + .addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(windowSize)); + SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> search.get()); + assertThat( + e.toString(), + containsString( + "Rescore window [" + + windowSize + + "] is too large. It must " + + "be less than [" + + IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + ) + ); + assertThat( + e.toString(), + containsString( + "This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting." + ) + ); + } } diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index 28931bb5a860f..960b46d68977b 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -896,6 +896,8 @@ public void evaluateRequestShouldUseConcurrentSearch() { && aggregations().factories() != null && !aggregations().factories().allFactoriesSupportConcurrentSearch()) { requestShouldUseConcurrentSearch.set(false); + } else if (terminateAfter != DEFAULT_TERMINATE_AFTER) { + requestShouldUseConcurrentSearch.set(false); } else { requestShouldUseConcurrentSearch.set(true); } From 2df9eed412c93c52ac404d6358bdc35685ee853f Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Tue, 3 Oct 2023 16:58:31 +0530 Subject: [PATCH 090/388] Add unreferenced file cleanup count to merge stats (#10204) Signed-off-by: Rishav Sagar Co-authored-by: Rishav Sagar --- CHANGELOG.md | 1 + .../org/opensearch/index/engine/Engine.java | 12 +++++++++- .../opensearch/index/merge/MergeStats.java | 22 +++++++++++++++++-- .../opensearch/index/shard/IndexShard.java | 4 +++- .../index/engine/InternalEngineTests.java | 10 ++++++++- 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9963eaef31d33..97a1386315dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131)) - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) - Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562)) +- Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index a8f2f60f8cf12..9b0e1b70a8e05 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -145,6 +145,7 @@ public abstract class Engine implements LifecycleAware, Closeable { protected final EngineConfig engineConfig; protected final Store store; protected final AtomicBoolean isClosed = new AtomicBoolean(false); + private final CounterMetric totalUnreferencedFileCleanUpsPerformed = new CounterMetric(); private final CountDownLatch closedLatch = new CountDownLatch(1); protected final EventListener eventListener; protected final ReentrantLock failEngineLock = new ReentrantLock(); @@ -267,6 +268,13 @@ protected final DocsStats docsStats(IndexReader indexReader) { return new DocsStats(numDocs, numDeletedDocs, sizeInBytes); } + /** + * Returns the unreferenced file cleanup count for this engine. + */ + public long unreferencedFileCleanUpsPerformed() { + return totalUnreferencedFileCleanUpsPerformed.count(); + } + /** * Performs the pre-closing checks on the {@link Engine}. * @@ -1340,7 +1348,9 @@ private void cleanUpUnreferencedFiles() { .setOpenMode(IndexWriterConfig.OpenMode.APPEND) ) ) { - // do nothing and close this will kick off IndexFileDeleter which will remove all unreferenced files. + // do nothing except increasing metric count and close this will kick off IndexFileDeleter which will + // remove all unreferenced files + totalUnreferencedFileCleanUpsPerformed.inc(); } catch (Exception ex) { logger.error("Error while deleting unreferenced file ", ex); } diff --git a/server/src/main/java/org/opensearch/index/merge/MergeStats.java b/server/src/main/java/org/opensearch/index/merge/MergeStats.java index 37fdca8871b18..fc5bac24a60d6 100644 --- a/server/src/main/java/org/opensearch/index/merge/MergeStats.java +++ b/server/src/main/java/org/opensearch/index/merge/MergeStats.java @@ -32,6 +32,7 @@ package org.opensearch.index.merge; +import org.opensearch.Version; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -65,9 +66,9 @@ public class MergeStats implements Writeable, ToXContentFragment { private long totalBytesPerSecAutoThrottle; - public MergeStats() { + private long unreferencedFileCleanUpsPerformed; - } + public MergeStats() {} public MergeStats(StreamInput in) throws IOException { total = in.readVLong(); @@ -81,6 +82,9 @@ public MergeStats(StreamInput in) throws IOException { totalStoppedTimeInMillis = in.readVLong(); totalThrottledTimeInMillis = in.readVLong(); totalBytesPerSecAutoThrottle = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + unreferencedFileCleanUpsPerformed = in.readOptionalVLong(); + } } public void add( @@ -133,6 +137,7 @@ public void addTotals(MergeStats mergeStats) { this.totalSizeInBytes += mergeStats.totalSizeInBytes; this.totalStoppedTimeInMillis += mergeStats.totalStoppedTimeInMillis; this.totalThrottledTimeInMillis += mergeStats.totalThrottledTimeInMillis; + addUnreferencedFileCleanUpStats(mergeStats.unreferencedFileCleanUpsPerformed); if (this.totalBytesPerSecAutoThrottle == Long.MAX_VALUE || mergeStats.totalBytesPerSecAutoThrottle == Long.MAX_VALUE) { this.totalBytesPerSecAutoThrottle = Long.MAX_VALUE; } else { @@ -140,6 +145,14 @@ public void addTotals(MergeStats mergeStats) { } } + public void addUnreferencedFileCleanUpStats(long unreferencedFileCleanUpsPerformed) { + this.unreferencedFileCleanUpsPerformed += unreferencedFileCleanUpsPerformed; + } + + public long getUnreferencedFileCleanUpsPerformed() { + return this.unreferencedFileCleanUpsPerformed; + } + /** * The total number of merges executed. */ @@ -240,6 +253,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.TOTAL_THROTTLE_BYTES_PER_SEC).value(new ByteSizeValue(totalBytesPerSecAutoThrottle).toString()); } builder.field(Fields.TOTAL_THROTTLE_BYTES_PER_SEC_IN_BYTES, totalBytesPerSecAutoThrottle); + builder.field(Fields.UNREFERENCED_FILE_CLEANUPS_PERFORMED, unreferencedFileCleanUpsPerformed); builder.endObject(); return builder; } @@ -267,6 +281,7 @@ static final class Fields { static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes"; static final String TOTAL_THROTTLE_BYTES_PER_SEC_IN_BYTES = "total_auto_throttle_in_bytes"; static final String TOTAL_THROTTLE_BYTES_PER_SEC = "total_auto_throttle"; + static final String UNREFERENCED_FILE_CLEANUPS_PERFORMED = "unreferenced_file_cleanups_performed"; } @Override @@ -282,5 +297,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(totalStoppedTimeInMillis); out.writeVLong(totalThrottledTimeInMillis); out.writeVLong(totalBytesPerSecAutoThrottle); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalVLong(unreferencedFileCleanUpsPerformed); + } } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 5ce066b156775..bc9d839624740 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1383,7 +1383,9 @@ public MergeStats mergeStats() { if (engine == null) { return new MergeStats(); } - return engine.getMergeStats(); + final MergeStats mergeStats = engine.getMergeStats(); + mergeStats.addUnreferencedFileCleanUpStats(engine.unreferencedFileCleanUpsPerformed()); + return mergeStats; } public SegmentsStats segmentStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index e3d77d45861ac..efa13438cffcf 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -233,7 +233,6 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -3334,6 +3333,9 @@ public void onFailedEngine(String reason, Exception e) { ); assertTrue(cleanupCompleted.await(10, TimeUnit.SECONDS)); + // Cleanup count will be incremented whenever cleanup is performed correctly. + long unreferencedFileCleanUpsPerformed = engine.unreferencedFileCleanUpsPerformed(); + assertThat(unreferencedFileCleanUpsPerformed, equalTo(1L)); } catch (Exception ex) { throw new AssertionError(ex); } @@ -3445,6 +3447,9 @@ public void onFailedEngine(String reason, Exception e) { ); assertTrue(cleanupCompleted.await(10, TimeUnit.SECONDS)); + // Cleanup count will not be incremented whenever cleanup is disabled. + long unreferencedFileCleanUpsPerformed = engine.unreferencedFileCleanUpsPerformed(); + assertThat(unreferencedFileCleanUpsPerformed, equalTo(0L)); } catch (Exception ex) { throw new AssertionError(ex); } @@ -3549,6 +3554,9 @@ public void onFailedEngine(String reason, Exception e) { ); assertTrue(cleanupCompleted.await(10, TimeUnit.SECONDS)); + // Cleanup count will not be incremented whenever there is some issue with cleanup. + long unreferencedFileCleanUpsPerformed = engine.unreferencedFileCleanUpsPerformed(); + assertThat(unreferencedFileCleanUpsPerformed, equalTo(0L)); } catch (Exception ex) { throw new AssertionError(ex); } From 20989755b2660ce9674fe2d2d95b125a6ffbfc7d Mon Sep 17 00:00:00 2001 From: Jongwoo Han Date: Tue, 3 Oct 2023 20:51:34 +0900 Subject: [PATCH 091/388] Replace deprecated command with environment file (#10289) Signed-off-by: Jongwoo Han --- .github/workflows/lucene-snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lucene-snapshots.yml b/.github/workflows/lucene-snapshots.yml index c2a2cedaaefb4..76981276fe085 100644 --- a/.github/workflows/lucene-snapshots.yml +++ b/.github/workflows/lucene-snapshots.yml @@ -38,7 +38,7 @@ jobs: - name: Set hash working-directory: ./lucene run: | - echo "::set-output name=REVISION::$(git rev-parse --short HEAD)" + echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT id: version - name: Initialize gradle settings From fe8068208c6243f1d6b1c22607461d6aa6f6893e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:27:53 -0400 Subject: [PATCH 092/388] Bump peter-evans/create-pull-request from 3 to 5 (#10301) * Bump peter-evans/create-pull-request from 3 to 5 Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/version.yml | 6 +++--- CHANGELOG.md | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index df785bcc70014..a20c671c137b2 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -59,7 +59,7 @@ jobs: sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v5 with: token: ${{ steps.github_app_token.outputs.token }} base: ${{ env.BASE }} @@ -86,7 +86,7 @@ jobs: sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v5 with: token: ${{ steps.github_app_token.outputs.token }} base: ${{ env.BASE_X }} @@ -113,7 +113,7 @@ jobs: sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v5 with: token: ${{ steps.github_app_token.outputs.token }} base: main diff --git a/CHANGELOG.md b/CHANGELOG.md index 97a1386315dc7..a6b9ff4eacfa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump Lucene from 9.7.0 to 9.8.0 ([#10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) - Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302)) - Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303)) +- Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) @@ -139,4 +140,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file From e7909fc45da3b4e60ea18a0b37563050d3af1d54 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Tue, 3 Oct 2023 20:07:08 +0530 Subject: [PATCH 093/388] Add TODO for updating public documentation (#10242) Signed-off-by: Bhumika Saini --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 69616e533d1ed..cd7c1bb980eec 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -19,6 +19,7 @@ Resolves #[Issue number to be closed when this PR is merged] - [ ] New functionality has javadoc added - [ ] Commits are signed per the DCO using --signoff - [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog)) +- [ ] GitHub issue/PR created in [OpenSearch documentation repo](https://github.com/opensearch-project/documentation-website) for the required public documentation changes (#[Issue/PR number]) By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). From 5aa18636706597351aacfa7fc51364cd54ba2c04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 11:11:27 -0400 Subject: [PATCH 094/388] Bump com.netflix.nebula.ospackage-base from 11.4.0 to 11.5.0 in /distribution/packages (#10295) * Bump com.netflix.nebula.ospackage-base in /distribution/packages Bumps com.netflix.nebula.ospackage-base from 11.4.0 to 11.5.0. --- updated-dependencies: - dependency-name: com.netflix.nebula.ospackage-base dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Andriy Redko --- CHANGELOG.md | 9 +++++---- distribution/packages/build.gradle | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6b9ff4eacfa2..8a37866012ba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,10 +106,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206)) - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208)) - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) -- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` -- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` -- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` -- Bump Lucene from 9.7.0 to 9.8.0 ([#10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) +- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` +- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) +- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295)) - Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302)) - Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303)) - Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301)) diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 262ad6c802bbb..cb05661dc74a4 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -63,7 +63,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.4.0" + id "com.netflix.nebula.ospackage-base" version "11.5.0" } void addProcessFilesTask(String type, boolean jdk) { From bddf0d3762f55d959ae76f3dea06259e7e795ebe Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 3 Oct 2023 16:07:34 -0400 Subject: [PATCH 095/388] Fix the HttpChannel trace termination in case of exceptions (#10325) Signed-off-by: Andriy Redko --- .../org/opensearch/telemetry/tracing/DefaultTracer.java | 5 ++++- .../org/opensearch/http/AbstractHttpServerTransport.java | 1 + server/src/main/java/org/opensearch/http/HttpChannel.java | 5 +++++ .../telemetry/tracing/channels/TraceableHttpChannel.java | 7 +++++++ .../tracing/handler/TraceableTransportResponseHandler.java | 1 + .../org/opensearch/transport/TransportResponseHandler.java | 2 +- 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java index d3c28b3a9cb5e..79b7e4aca6c2f 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java @@ -25,7 +25,10 @@ */ @InternalApi class DefaultTracer implements Tracer { - static final String THREAD_NAME = "th_name"; + /** + * Current thread name. + */ + static final String THREAD_NAME = "thread.name"; private final TracingTelemetry tracingTelemetry; private final TracerContextStorage tracerContextStorage; diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index ed44102d0abe4..b8f8abb6c2c23 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -298,6 +298,7 @@ static int resolvePublishPort(Settings settings, List boundAdd } public void onException(HttpChannel channel, Exception e) { + channel.handleException(e); if (lifecycle.started() == false) { // just close and ignore - we are already stopped and just need to make sure we release all resources CloseableChannel.closeChannel(channel); diff --git a/server/src/main/java/org/opensearch/http/HttpChannel.java b/server/src/main/java/org/opensearch/http/HttpChannel.java index 99aaed23c69b8..6dcdaf9034413 100644 --- a/server/src/main/java/org/opensearch/http/HttpChannel.java +++ b/server/src/main/java/org/opensearch/http/HttpChannel.java @@ -43,6 +43,11 @@ * @opensearch.internal */ public interface HttpChannel extends CloseableChannel { + /** + * Notify HTTP channel that exception happens and the response may not be sent (for example, timeout) + * @param ex the exception being raised + */ + default void handleException(Exception ex) {} /** * Sends an http response to the channel. The listener will be executed once the send process has been diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java index 9229d334dea01..03848e8e58207 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java @@ -56,6 +56,13 @@ public static HttpChannel create(HttpChannel delegate, Span span, Tracer tracer) } } + @Override + public void handleException(Exception ex) { + span.addEvent("The HttpChannel was closed without sending the response"); + span.setError(ex); + span.endSpan(); + } + @Override public void close() { delegate.close(); diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java index abddfcc6cebc1..538bf82a1dbec 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java @@ -101,6 +101,7 @@ public void handleRejection(Exception exp) { try (SpanScope scope = tracer.withSpanInScope(span)) { delegate.handleRejection(exp); } finally { + span.setError(exp); span.endSpan(); } } diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java index 90e94e52515ce..8992af18edb48 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java @@ -57,7 +57,7 @@ public interface TransportResponseHandler extends W * It should be used to clear up the resources held by the {@link TransportResponseHandler}. * @param exp exception */ - default void handleRejection(Exception exp) {}; + default void handleRejection(Exception exp) {} default TransportResponseHandler wrap(Function converter, Writeable.Reader reader) { final TransportResponseHandler self = this; From d5a95b81045828cfbd4926a10685eb03a83457c0 Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Wed, 4 Oct 2023 03:40:21 +0530 Subject: [PATCH 096/388] Changing unrefernced file cleanup stats based MergeStats metrics version to 2.11 (#10336) Signed-off-by: Rishav Sagar Co-authored-by: Rishav Sagar --- .../src/main/java/org/opensearch/index/merge/MergeStats.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/merge/MergeStats.java b/server/src/main/java/org/opensearch/index/merge/MergeStats.java index fc5bac24a60d6..a284cec247ff1 100644 --- a/server/src/main/java/org/opensearch/index/merge/MergeStats.java +++ b/server/src/main/java/org/opensearch/index/merge/MergeStats.java @@ -82,7 +82,7 @@ public MergeStats(StreamInput in) throws IOException { totalStoppedTimeInMillis = in.readVLong(); totalThrottledTimeInMillis = in.readVLong(); totalBytesPerSecAutoThrottle = in.readVLong(); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_11_0)) { unreferencedFileCleanUpsPerformed = in.readOptionalVLong(); } } @@ -297,7 +297,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(totalStoppedTimeInMillis); out.writeVLong(totalThrottledTimeInMillis); out.writeVLong(totalBytesPerSecAutoThrottle); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_11_0)) { out.writeOptionalVLong(unreferencedFileCleanUpsPerformed); } } From 7159e2e36ab00906ae83c50001fce41347ac514b Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:53:53 +0530 Subject: [PATCH 097/388] Add node id to segment and translog metadata (#10229) * Adding node id to segment and translog metadata Adding validation to identify multiple writers to same primary term and generation in remote store Signed-off-by: Gaurav Bafna * Fix bug to find node id Signed-off-by: Gaurav Bafna * Spotless fix Signed-off-by: Gaurav Bafna * Moving node id before uuid as it was interfering with _ character split Signed-off-by: Gaurav Bafna * Removing uuid from segment metadata file Signed-off-by: Gaurav Bafna * simplifying the detection logic Signed-off-by: Gaurav Bafna * PR comments Signed-off-by: Gaurav Bafna * Addressing PR comments Signed-off-by: Gaurav Bafna * Adding translog gen to remote segment metadata fn as well Signed-off-by: Gaurav Bafna * spotless fix Signed-off-by: Gaurav Bafna * reducing METADATA_FILES_TO_FETCH to 10 Signed-off-by: Gaurav Bafna * adding missing import Signed-off-by: Gaurav Bafna --------- Signed-off-by: Gaurav Bafna --- .../opensearch/index/shard/IndexShardIT.java | 14 ++- .../org/opensearch/index/IndexService.java | 3 +- .../index/remote/RemoteStoreUtils.java | 33 +++++ .../opensearch/index/shard/IndexShard.java | 9 +- .../shard/RemoteStoreRefreshListener.java | 3 +- .../store/RemoteSegmentStoreDirectory.java | 33 ++++- .../index/translog/RemoteFsTranslog.java | 3 +- .../index/translog/TranslogConfig.java | 19 ++- .../translog/TruncateTranslogAction.java | 3 +- .../TranslogCheckpointTransferSnapshot.java | 21 +++- .../transfer/TranslogTransferManager.java | 12 +- .../transfer/TranslogTransferMetadata.java | 37 +++++- .../index/engine/InternalEngineTests.java | 8 +- .../index/remote/RemoteStoreUtilsTests.java | 119 ++++++++++++++++++ .../index/shard/RefreshListenersTests.java | 3 +- .../RemoteStoreRefreshListenerTests.java | 10 +- ...moteSegmentStoreDirectoryFactoryTests.java | 10 +- .../RemoteSegmentStoreDirectoryTests.java | 87 +++++++++---- .../InternalTranslogManagerTests.java | 14 +-- .../index/translog/LocalTranslogTests.java | 8 +- .../index/translog/RemoteFsTranslogTests.java | 8 +- .../translog/TranslogManagerTestCase.java | 2 +- .../TranslogTransferManagerTests.java | 49 +++++++- .../index/engine/EngineTestCase.java | 12 +- .../index/shard/IndexShardTestCase.java | 3 +- 25 files changed, 445 insertions(+), 78 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index bb08b19df765b..07f85496f13cf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -650,7 +650,15 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResul } } }; - final IndexShard newShard = newIndexShard(indexService, shard, wrapper, getInstanceFromNode(CircuitBreakerService.class), listener); + NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class); + final IndexShard newShard = newIndexShard( + indexService, + shard, + wrapper, + getInstanceFromNode(CircuitBreakerService.class), + env.nodeId(), + listener + ); shardRef.set(newShard); recoverShard(newShard); @@ -674,6 +682,7 @@ public static final IndexShard newIndexShard( final IndexShard shard, CheckedFunction wrapper, final CircuitBreakerService cbs, + final String nodeId, final IndexingOperationListener... listeners ) throws IOException { ShardRouting initializingShardRouting = getInitializingShardRouting(shard.routingEntry()); @@ -702,7 +711,8 @@ public static final IndexShard newIndexShard( SegmentReplicationCheckpointPublisher.EMPTY, null, null, - () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, + nodeId ); } diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index ca0cc307e460b..fdda8d4ce2497 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -519,7 +519,8 @@ public synchronized IndexShard createShard( this.indexSettings.isSegRepEnabled() ? checkpointPublisher : null, remoteStore, remoteStoreStatsTrackerFactory, - clusterRemoteTranslogBufferIntervalSupplier + clusterRemoteTranslogBufferIntervalSupplier, + nodeEnv.nodeId() ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index 114d07589b0c0..0ca9e0209c5ec 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -8,7 +8,13 @@ package org.opensearch.index.remote; +import org.opensearch.common.collect.Tuple; + import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; /** * Utils for remote store @@ -69,4 +75,31 @@ public static String getSegmentName(String filename) { return filename.substring(0, endIdx); } + + /** + * + * @param mdFiles List of segment/translog metadata files + * @param fn Function to extract PrimaryTerm_Generation and Node Id from metadata file name . + * fn returns null if node id is not part of the file name + */ + static public void verifyNoMultipleWriters(List mdFiles, Function> fn) { + Map nodesByPrimaryTermAndGen = new HashMap<>(); + mdFiles.forEach(mdFile -> { + Tuple nodeIdByPrimaryTermAndGen = fn.apply(mdFile); + if (nodeIdByPrimaryTermAndGen != null) { + if (nodesByPrimaryTermAndGen.containsKey(nodeIdByPrimaryTermAndGen.v1()) + && (!nodesByPrimaryTermAndGen.get(nodeIdByPrimaryTermAndGen.v1()).equals(nodeIdByPrimaryTermAndGen.v2()))) { + throw new IllegalStateException( + "Multiple metadata files from different nodes" + + nodeIdByPrimaryTermAndGen.v1() + + " and " + + nodeIdByPrimaryTermAndGen.v2() + + "having same primary term and generations detected" + ); + } + nodesByPrimaryTermAndGen.put(nodeIdByPrimaryTermAndGen.v1(), nodeIdByPrimaryTermAndGen.v2()); + } + }); + } + } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index bc9d839624740..5818b2d866854 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -367,7 +367,8 @@ public IndexShard( @Nullable final SegmentReplicationCheckpointPublisher checkpointPublisher, @Nullable final Store remoteStore, final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, - final Supplier clusterRemoteTranslogBufferIntervalSupplier + final Supplier clusterRemoteTranslogBufferIntervalSupplier, + final String nodeId ) throws IOException { super(shardRouting.shardId(), indexSettings); assert shardRouting.initializing(); @@ -413,7 +414,7 @@ public IndexShard( logger.debug("state: [CREATED]"); this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP); - this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays); + this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays, nodeId); final String aId = shardRouting.allocationId().getId(); final long primaryTerm = indexSettings.getIndexMetadata().primaryTerm(shardId.id()); this.pendingPrimaryTerm = primaryTerm; @@ -556,6 +557,10 @@ protected RemoteStoreStatsTrackerFactory getRemoteStoreStatsTrackerFactory() { return remoteStoreStatsTrackerFactory; } + public String getNodeId() { + return translogConfig.getNodeId(); + } + @Override public void updateShardState( final ShardRouting newRouting, diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 695c01367171a..698e61f6f7a09 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -346,7 +346,8 @@ void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos se segmentInfosSnapshot, storeDirectory, translogFileGeneration, - replicationCheckpoint + replicationCheckpoint, + indexShard.getNodeId() ); } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index b23d2d7d0a3f8..21a84f2b8c903 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -25,6 +25,7 @@ import org.apache.lucene.util.Version; import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.logging.Loggers; import org.opensearch.common.lucene.store.ByteArrayIndexInput; @@ -114,6 +115,8 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement private final AtomicLong metadataUploadCounter = new AtomicLong(0); + public static final int METADATA_FILES_TO_FETCH = 10; + public RemoteSegmentStoreDirectory( RemoteDirectory remoteDataDirectory, RemoteDirectory remoteMetadataDirectory, @@ -187,9 +190,11 @@ public RemoteSegmentMetadata readLatestMetadataFile() throws IOException { List metadataFiles = remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ); + RemoteStoreUtils.verifyNoMultipleWriters(metadataFiles, MetadataFilenameUtils::getNodeIdByPrimaryTermAndGen); + if (metadataFiles.isEmpty() == false) { String latestMetadataFile = metadataFiles.get(0); logger.trace("Reading latest Metadata file {}", latestMetadataFile); @@ -306,12 +311,13 @@ static String getMetadataFilePrefixForCommit(long primaryTerm, long generation) } // Visible for testing - static String getMetadataFilename( + public static String getMetadataFilename( long primaryTerm, long generation, long translogGeneration, long uploadCounter, - int metadataVersion + int metadataVersion, + String nodeId ) { return String.join( SEPARATOR, @@ -320,6 +326,7 @@ static String getMetadataFilename( RemoteStoreUtils.invertLong(generation), RemoteStoreUtils.invertLong(translogGeneration), RemoteStoreUtils.invertLong(uploadCounter), + nodeId, RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(metadataVersion) ); @@ -334,6 +341,19 @@ static long getPrimaryTerm(String[] filenameTokens) { static long getGeneration(String[] filenameTokens) { return RemoteStoreUtils.invertLong(filenameTokens[2]); } + + public static Tuple getNodeIdByPrimaryTermAndGen(String filename) { + String[] tokens = filename.split(SEPARATOR); + if (tokens.length < 8) { + // For versions < 2.11, we don't have node id. + return null; + } + String primaryTermAndGen = String.join(SEPARATOR, tokens[1], tokens[2], tokens[3]); + + String nodeId = tokens[5]; + return new Tuple<>(primaryTermAndGen, nodeId); + } + } /** @@ -593,6 +613,7 @@ public boolean containsFile(String localFilename, String checksum) { * @param storeDirectory instance of local directory to temporarily create metadata file before upload * @param translogGeneration translog generation * @param replicationCheckpoint ReplicationCheckpoint of primary shard + * @param nodeId node id * @throws IOException in case of I/O error while uploading the metadata file */ public void uploadMetadata( @@ -600,7 +621,8 @@ public void uploadMetadata( SegmentInfos segmentInfosSnapshot, Directory storeDirectory, long translogGeneration, - ReplicationCheckpoint replicationCheckpoint + ReplicationCheckpoint replicationCheckpoint, + String nodeId ) throws IOException { synchronized (this) { String metadataFilename = MetadataFilenameUtils.getMetadataFilename( @@ -608,7 +630,8 @@ public void uploadMetadata( segmentInfosSnapshot.getGeneration(), translogGeneration, metadataUploadCounter.incrementAndGet(), - RemoteSegmentMetadata.CURRENT_VERSION + RemoteSegmentMetadata.CURRENT_VERSION, + nodeId ); try { try (IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT)) { diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 004d1bfdca36d..857d90e2e2ac2 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -327,7 +327,8 @@ private boolean upload(Long primaryTerm, Long generation) throws IOException { generation, location, readers, - Translog::getCommitCheckpointFileName + Translog::getCommitCheckpointFileName, + config.getNodeId() ).build() ) { return translogTransferManager.transferSnapshot( diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java index cac88bee82a73..6e75ebd847b5e 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java @@ -56,6 +56,7 @@ public final class TranslogConfig { private final ShardId shardId; private final Path translogPath; private final ByteSizeValue bufferSize; + private final String nodeId; /** * Creates a new TranslogConfig instance @@ -64,16 +65,24 @@ public final class TranslogConfig { * @param indexSettings the index settings used to set internal variables * @param bigArrays a bigArrays instance used for temporarily allocating write operations */ - public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSettings, BigArrays bigArrays) { - this(shardId, translogPath, indexSettings, bigArrays, DEFAULT_BUFFER_SIZE); + public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSettings, BigArrays bigArrays, String nodeId) { + this(shardId, translogPath, indexSettings, bigArrays, DEFAULT_BUFFER_SIZE, nodeId); } - TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSettings, BigArrays bigArrays, ByteSizeValue bufferSize) { + TranslogConfig( + ShardId shardId, + Path translogPath, + IndexSettings indexSettings, + BigArrays bigArrays, + ByteSizeValue bufferSize, + String nodeId + ) { this.bufferSize = bufferSize; this.indexSettings = indexSettings; this.shardId = shardId; this.translogPath = translogPath; this.bigArrays = bigArrays; + this.nodeId = nodeId; } /** @@ -110,4 +119,8 @@ public Path getTranslogPath() { public ByteSizeValue getBufferSize() { return bufferSize; } + + public String getNodeId() { + return nodeId; + } } diff --git a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java index 0d85123b60c75..25fcdc614172a 100644 --- a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java +++ b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java @@ -194,7 +194,8 @@ private boolean isTranslogClean(ShardPath shardPath, ClusterState clusterState, shardPath.getShardId(), translogPath, indexSettings, - BigArrays.NON_RECYCLING_INSTANCE + BigArrays.NON_RECYCLING_INSTANCE, + "" ); long primaryTerm = indexSettings.getIndexMetadata().primaryTerm(shardPath.getShardId().id()); // We open translog to check for corruption, do not clean anything. diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java index 10dec13c81e1a..fb78731246a07 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogCheckpointTransferSnapshot.java @@ -40,11 +40,14 @@ public class TranslogCheckpointTransferSnapshot implements TransferSnapshot, Clo private final long primaryTerm; private long minTranslogGeneration; - TranslogCheckpointTransferSnapshot(long primaryTerm, long generation, int size) { + private String nodeId; + + TranslogCheckpointTransferSnapshot(long primaryTerm, long generation, int size, String nodeId) { translogCheckpointFileInfoTupleSet = new HashSet<>(size); this.size = size; this.generation = generation; this.primaryTerm = primaryTerm; + this.nodeId = nodeId; } private void add(TranslogFileSnapshot translogFileSnapshot, CheckpointFileSnapshot checkPointFileSnapshot) { @@ -63,7 +66,13 @@ public Set getTranslogFileSnapshots() { @Override public TranslogTransferMetadata getTranslogTransferMetadata() { - return new TranslogTransferMetadata(primaryTerm, generation, minTranslogGeneration, translogCheckpointFileInfoTupleSet.size() * 2); + return new TranslogTransferMetadata( + primaryTerm, + generation, + minTranslogGeneration, + translogCheckpointFileInfoTupleSet.size() * 2, + nodeId + ); } @Override @@ -110,19 +119,22 @@ public static class Builder { private final List readers; private final Function checkpointGenFileNameMapper; private final Path location; + private final String nodeId; public Builder( long primaryTerm, long generation, Path location, List readers, - Function checkpointGenFileNameMapper + Function checkpointGenFileNameMapper, + String nodeId ) { this.primaryTerm = primaryTerm; this.generation = generation; this.readers = readers; this.checkpointGenFileNameMapper = checkpointGenFileNameMapper; this.location = location; + this.nodeId = nodeId; } public TranslogCheckpointTransferSnapshot build() throws IOException { @@ -134,7 +146,8 @@ public TranslogCheckpointTransferSnapshot build() throws IOException { TranslogCheckpointTransferSnapshot translogTransferSnapshot = new TranslogCheckpointTransferSnapshot( primaryTerm, generation, - readers.size() + readers.size(), + nodeId ); for (TranslogReader reader : readers) { final long readerGeneration = reader.getGeneration(); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index d7e50cdcf32e4..d988b8a6254ff 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -24,6 +24,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.transfer.listener.TranslogTransferListener; @@ -64,6 +65,8 @@ public class TranslogTransferManager { private static final long TRANSFER_TIMEOUT_IN_MILLIS = 30000; + private static final int METADATA_FILES_TO_FETCH = 10; + private final Logger logger; private final static String METADATA_DIR = "metadata"; private final static String DATA_DIR = "data"; @@ -275,6 +278,10 @@ public TranslogTransferMetadata readMetadata() throws IOException { LatchedActionListener> latchedActionListener = new LatchedActionListener<>( ActionListener.wrap(blobMetadataList -> { if (blobMetadataList.isEmpty()) return; + RemoteStoreUtils.verifyNoMultipleWriters( + blobMetadataList.stream().map(BlobMetadata::name).collect(Collectors.toList()), + TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen + ); String filename = blobMetadataList.get(0).name(); boolean downloadStatus = false; long downloadStartTime = System.nanoTime(), bytesToRead = 0; @@ -295,6 +302,9 @@ public TranslogTransferMetadata readMetadata() throws IOException { } } }, e -> { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } logger.error(() -> new ParameterizedMessage("Exception while listing metadata files"), e); exceptionSetOnce.set((IOException) e); }), @@ -305,7 +315,7 @@ public TranslogTransferMetadata readMetadata() throws IOException { transferService.listAllInSortedOrder( remoteMetadataTransferPath, TranslogTransferMetadata.METADATA_PREFIX, - 1, + METADATA_FILES_TO_FETCH, latchedActionListener ); latch.await(); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index a8b3404d3f2ce..07ec9160db1e3 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -9,6 +9,7 @@ package org.opensearch.index.translog.transfer; import org.opensearch.common.SetOnce; +import org.opensearch.common.collect.Tuple; import org.opensearch.index.remote.RemoteStoreUtils; import java.util.Arrays; @@ -30,7 +31,7 @@ public class TranslogTransferMetadata { private final long minTranslogGeneration; - private int count; + private final int count; private final SetOnce> generationToPrimaryTermMapper = new SetOnce<>(); @@ -46,12 +47,22 @@ public class TranslogTransferMetadata { private final long createdAt; - public TranslogTransferMetadata(long primaryTerm, long generation, long minTranslogGeneration, int count) { + private final String nodeId; + + public TranslogTransferMetadata(long primaryTerm, long generation, long minTranslogGeneration, int count, String nodeId) { this.primaryTerm = primaryTerm; this.generation = generation; this.minTranslogGeneration = minTranslogGeneration; this.count = count; this.createdAt = System.currentTimeMillis(); + this.nodeId = nodeId; + } + + /* + Used only at the time of download . Since details are read from content , nodeId is not available + */ + public TranslogTransferMetadata(long primaryTerm, long generation, long minTranslogGeneration, int count) { + this(primaryTerm, generation, minTranslogGeneration, count, ""); } public long getPrimaryTerm() { @@ -89,11 +100,33 @@ public String getFileName() { RemoteStoreUtils.invertLong(primaryTerm), RemoteStoreUtils.invertLong(generation), RemoteStoreUtils.invertLong(createdAt), + nodeId, String.valueOf(CURRENT_VERSION) ) ); } + public static Tuple, String> getNodeIdByPrimaryTermAndGeneration(String filename) { + String[] tokens = filename.split(METADATA_SEPARATOR); + if (tokens.length < 6) { + // For versions < 2.11, we don't have node id + return null; + } + return new Tuple<>(new Tuple<>(RemoteStoreUtils.invertLong(tokens[1]), RemoteStoreUtils.invertLong(tokens[2])), tokens[4]); + } + + public static Tuple getNodeIdByPrimaryTermAndGen(String filename) { + String[] tokens = filename.split(METADATA_SEPARATOR); + if (tokens.length < 6) { + // For versions < 2.11, we don't have node id. + return null; + } + String primaryTermAndGen = String.join(METADATA_SEPARATOR, tokens[1], tokens[2]); + + String nodeId = tokens[4]; + return new Tuple<>(primaryTermAndGen, nodeId); + } + @Override public int hashCode() { return Objects.hash(primaryTerm, generation); diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index efa13438cffcf..b8bb73bb89a82 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -4005,7 +4005,7 @@ public void testRecoverFromForeignTranslog() throws IOException { final Path badTranslogLog = createTempDir(); final String badUUID = Translog.createEmptyTranslog(badTranslogLog, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); Translog translog = new LocalTranslog( - new TranslogConfig(shardId, badTranslogLog, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, badTranslogLog, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), badUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), () -> SequenceNumbers.NO_OPS_PERFORMED, @@ -4022,7 +4022,8 @@ public void testRecoverFromForeignTranslog() throws IOException { shardId, translog.location(), config.getIndexSettings(), - BigArrays.NON_RECYCLING_INSTANCE + BigArrays.NON_RECYCLING_INSTANCE, + "" ); EngineConfig brokenConfig = new EngineConfig.Builder().shardId(shardId) @@ -7711,7 +7712,8 @@ public void testNotWarmUpSearcherInEngineCtor() throws Exception { config.getTranslogConfig().getShardId(), createTempDir(), config.getTranslogConfig().getIndexSettings(), - config.getTranslogConfig().getBigArrays() + config.getTranslogConfig().getBigArrays(), + "" ); EngineConfig configWithWarmer = new EngineConfig.Builder().shardId(config.getShardId()) .threadPool(config.getThreadPool()) diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java index 9afa75dd601b2..181c846d394a0 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -8,10 +8,85 @@ package org.opensearch.index.remote; +import org.opensearch.common.blobstore.BlobMetadata; +import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.translog.transfer.TranslogTransferMetadata; import org.opensearch.test.OpenSearchTestCase; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.index.remote.RemoteStoreUtils.verifyNoMultipleWriters; +import static org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX; +import static org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR; +import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; + public class RemoteStoreUtilsTests extends OpenSearchTestCase { + private final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename( + 12, + 23, + 34, + 1, + 1, + "node-1" + ); + + private final String metadataFilenameDup = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename( + 12, + 23, + 34, + 2, + 1, + "node-2" + ); + private final String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename( + 12, + 13, + 34, + 1, + 1, + "node-1" + ); + + private final String oldMetadataFilename = getOldSegmentMetadataFilename(12, 23, 34, 1, 1); + + /* + Gives segment metadata filename for <2.11 version + */ + public static String getOldSegmentMetadataFilename( + long primaryTerm, + long generation, + long translogGeneration, + long uploadCounter, + int metadataVersion + ) { + return String.join( + SEPARATOR, + METADATA_PREFIX, + RemoteStoreUtils.invertLong(primaryTerm), + RemoteStoreUtils.invertLong(generation), + RemoteStoreUtils.invertLong(translogGeneration), + RemoteStoreUtils.invertLong(uploadCounter), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(metadataVersion) + ); + } + + public static String getOldTranslogMetadataFilename(long primaryTerm, long generation, int metadataVersion) { + return String.join( + METADATA_SEPARATOR, + METADATA_PREFIX, + RemoteStoreUtils.invertLong(primaryTerm), + RemoteStoreUtils.invertLong(generation), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(metadataVersion) + ); + } + public void testInvertToStrInvalid() { assertThrows(IllegalArgumentException.class, () -> RemoteStoreUtils.invertLong(-1)); } @@ -60,4 +135,48 @@ public void testGetSegmentNameUnderscoreDelimiterOverrides() { public void testGetSegmentNameException() { assertThrows(IllegalArgumentException.class, () -> RemoteStoreUtils.getSegmentName("dvd")); } + + public void testVerifyMultipleWriters_Segment() { + List mdFiles = new ArrayList<>(); + mdFiles.add(metadataFilename); + mdFiles.add(metadataFilename2); + mdFiles.add(oldMetadataFilename); + verifyNoMultipleWriters(mdFiles, RemoteSegmentStoreDirectory.MetadataFilenameUtils::getNodeIdByPrimaryTermAndGen); + + mdFiles.add(metadataFilenameDup); + assertThrows( + IllegalStateException.class, + () -> verifyNoMultipleWriters(mdFiles, RemoteSegmentStoreDirectory.MetadataFilenameUtils::getNodeIdByPrimaryTermAndGen) + ); + } + + public void testVerifyMultipleWriters_Translog() throws InterruptedException { + TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2, "node-1"); + String mdFilename = tm.getFileName(); + Thread.sleep(1); + TranslogTransferMetadata tm2 = new TranslogTransferMetadata(1, 1, 1, 2, "node-1"); + String mdFilename2 = tm2.getFileName(); + List bmList = new LinkedList<>(); + bmList.add(new PlainBlobMetadata(mdFilename, 1)); + bmList.add(new PlainBlobMetadata(mdFilename2, 1)); + bmList.add(new PlainBlobMetadata(getOldTranslogMetadataFilename(1, 1, 1), 1)); + RemoteStoreUtils.verifyNoMultipleWriters( + bmList.stream().map(BlobMetadata::name).collect(Collectors.toList()), + TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen + ); + + bmList = new LinkedList<>(); + bmList.add(new PlainBlobMetadata(mdFilename, 1)); + TranslogTransferMetadata tm3 = new TranslogTransferMetadata(1, 1, 1, 2, "node-2"); + bmList.add(new PlainBlobMetadata(tm3.getFileName(), 1)); + List finalBmList = bmList; + assertThrows( + IllegalStateException.class, + () -> RemoteStoreUtils.verifyNoMultipleWriters( + finalBmList.stream().map(BlobMetadata::name).collect(Collectors.toList()), + TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen + ) + ); + } + } diff --git a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java index afe6e47bec7b2..a45b25f04060b 100644 --- a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java @@ -133,7 +133,8 @@ public void setupListeners() throws Exception { shardId, createTempDir("translog"), indexSettings, - BigArrays.NON_RECYCLING_INSTANCE + BigArrays.NON_RECYCLING_INSTANCE, + "" ); Engine.EventListener eventListener = new Engine.EventListener() { @Override diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 415efd4ac23b6..5a13f57db2c87 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -46,6 +46,7 @@ import java.util.concurrent.atomic.AtomicLong; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; +import static org.opensearch.index.store.RemoteSegmentStoreDirectory.METADATA_FILES_TO_FETCH; import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes; import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata; import static org.mockito.ArgumentMatchers.any; @@ -138,7 +139,8 @@ public void testRemoteDirectoryInitThrowsException() throws IOException { return Collections.singletonList("dummy string"); } throw new IOException(); - }).when(remoteMetadataDirectory).listFilesByPrefixInLexicographicOrder(MetadataFilenameUtils.METADATA_PREFIX, 1); + }).when(remoteMetadataDirectory) + .listFilesByPrefixInLexicographicOrder(MetadataFilenameUtils.METADATA_PREFIX, METADATA_FILES_TO_FETCH); SegmentInfos segmentInfos; try (Store indexShardStore = indexShard.store()) { @@ -166,7 +168,10 @@ public void testRemoteDirectoryInitThrowsException() throws IOException { // Validate that the stream of metadata file of remoteMetadataDirectory has been opened only once and the // listFilesByPrefixInLexicographicOrder has been called twice. verify(remoteMetadataDirectory, times(1)).getBlobStream(any()); - verify(remoteMetadataDirectory, times(2)).listFilesByPrefixInLexicographicOrder(MetadataFilenameUtils.METADATA_PREFIX, 1); + verify(remoteMetadataDirectory, times(2)).listFilesByPrefixInLexicographicOrder( + MetadataFilenameUtils.METADATA_PREFIX, + METADATA_FILES_TO_FETCH + ); } public void testAfterRefresh() throws IOException { @@ -579,4 +584,5 @@ private void verifyUploadedSegments(RemoteSegmentStoreDirectory remoteSegmentSto } } } + } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index d7bbe52aa3905..cad5e47531cc6 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -35,6 +35,7 @@ import org.mockito.ArgumentCaptor; +import static org.opensearch.index.store.RemoteSegmentStoreDirectory.METADATA_FILES_TO_FETCH; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; @@ -78,7 +79,12 @@ public void testNewDirectory() throws IOException { latchedActionListener.onResponse(List.of()); return null; }).when(blobContainer) - .listBlobsByPrefixInSortedOrder(any(), eq(1), eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), any(ActionListener.class)); + .listBlobsByPrefixInSortedOrder( + any(), + eq(METADATA_FILES_TO_FETCH), + eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), + any(ActionListener.class) + ); when(repositoriesService.repository("remote_store_repository")).thenReturn(repository); @@ -93,7 +99,7 @@ public void testNewDirectory() throws IOException { verify(blobContainer).listBlobsByPrefixInSortedOrder( eq(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX), - eq(1), + eq(METADATA_FILES_TO_FETCH), eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), any() ); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 8d99d98fbaaf4..f154dddb0e7cc 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -62,6 +62,7 @@ import org.mockito.Mockito; +import static org.opensearch.index.store.RemoteSegmentStoreDirectory.METADATA_FILES_TO_FETCH; import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes; import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata; import static org.hamcrest.CoreMatchers.is; @@ -90,9 +91,39 @@ public class RemoteSegmentStoreDirectoryTests extends IndexShardTestCase { private SegmentInfos segmentInfos; private ThreadPool threadPool; - private final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 23, 34, 1, 1); - private final String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 13, 34, 1, 1); - private final String metadataFilename3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(10, 38, 34, 1, 1); + private final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename( + 12, + 23, + 34, + 1, + 1, + "node-1" + ); + + private final String metadataFilenameDup = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename( + 12, + 23, + 34, + 2, + 1, + "node-2" + ); + private final String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename( + 12, + 13, + 34, + 1, + 1, + "node-1" + ); + private final String metadataFilename3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename( + 10, + 38, + 34, + 1, + 1, + "node-1" + ); @Before public void setup() throws IOException { @@ -183,7 +214,7 @@ public void testGetPrimaryTermGenerationUuid() { } public void testInitException() throws IOException { - when(remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, 1)).thenThrow( + when(remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, METADATA_FILES_TO_FETCH)).thenThrow( new IOException("Error") ); @@ -202,6 +233,13 @@ public void testInitNoMetadataFile() throws IOException { assertEquals(Set.of(), actualCache.keySet()); } + public void testInitMultipleMetadataFile() throws IOException { + when(remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, METADATA_FILES_TO_FETCH)).thenReturn( + List.of(metadataFilename, metadataFilenameDup) + ); + assertThrows(IllegalStateException.class, () -> remoteSegmentStoreDirectory.init()); + } + private Map> populateMetadata() throws IOException { List metadataFiles = new ArrayList<>(); @@ -212,7 +250,7 @@ private Map> populateMetadata() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(List.of(metadataFilename)); when( @@ -262,7 +300,7 @@ public void testInit() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(List.of(metadataFilename)); @@ -318,7 +356,7 @@ public void testFileLength() throws IOException { assertEquals(uploadedSegments.get("_0.si").getLength(), remoteSegmentStoreDirectory.fileLength("_0.si")); } - public void testFileLenghtNoSuchFile() throws IOException { + public void testFileLengthNoSuchFile() throws IOException { populateMetadata(); remoteSegmentStoreDirectory.init(); @@ -693,7 +731,7 @@ public void testContainsFile() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(metadataFiles); @@ -739,7 +777,8 @@ public void testUploadMetadataEmpty() throws IOException { segmentInfos, storeDirectory, 34L, - indexShard.getLatestReplicationCheckpoint() + indexShard.getLatestReplicationCheckpoint(), + "" ) ); } @@ -757,7 +796,7 @@ public void testUploadMetadataNonEmpty() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(metadataFiles); Map> metadataFilenameContentMapping = Map.of( @@ -785,7 +824,8 @@ public void testUploadMetadataNonEmpty() throws IOException { segInfos, storeDirectory, generation, - indexShard.getLatestReplicationCheckpoint() + indexShard.getLatestReplicationCheckpoint(), + "" ); verify(remoteMetadataDirectory).copyFrom( @@ -832,7 +872,8 @@ public void testUploadMetadataMissingSegment() throws IOException { segmentInfos, storeDirectory, 12L, - indexShard.getLatestReplicationCheckpoint() + indexShard.getLatestReplicationCheckpoint(), + "" ) ); verify(indexOutput).close(); @@ -855,7 +896,7 @@ public void testNoMetadataHeaderCorruptIndexException() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(metadataFiles); @@ -878,7 +919,7 @@ public void testInvalidCodecHeaderCorruptIndexException() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(metadataFiles); @@ -903,7 +944,7 @@ public void testHeaderMinVersionCorruptIndexException() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(metadataFiles); @@ -928,7 +969,7 @@ public void testHeaderMaxVersionCorruptIndexException() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(metadataFiles); @@ -953,7 +994,7 @@ public void testIncorrectChecksumCorruptIndexException() throws IOException { when( remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder( RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, - 1 + METADATA_FILES_TO_FETCH ) ).thenReturn(metadataFiles); @@ -1114,12 +1155,12 @@ private void indexDocs(int startDocId, int numberOfDocs) throws IOException { } public void testMetadataFileNameOrder() { - String file1 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 21, 23, 1, 1); - String file2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 38, 1, 1); - String file3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(18, 12, 26, 1, 1); - String file4 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 10, 1); - String file5 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 1, 1); - String file6 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 5, 1); + String file1 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 21, 23, 1, 1, ""); + String file2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 38, 1, 1, ""); + String file3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(18, 12, 26, 1, 1, ""); + String file4 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 10, 1, ""); + String file5 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 1, 1, ""); + String file6 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 32, 5, 1, ""); List actualList = new ArrayList<>(List.of(file1, file2, file3, file4, file5, file6)); actualList.sort(String::compareTo); diff --git a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java index 2de36574064cb..c098d11a3487f 100644 --- a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java @@ -38,7 +38,7 @@ public void testRecoveryFromTranslog() throws IOException { LocalCheckpointTracker tracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED); try { translogManager = new InternalTranslogManager( - new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), primaryTerm, globalCheckpoint::get, createTranslogDeletionPolicy(INDEX_SETTINGS), @@ -68,7 +68,7 @@ public void testRecoveryFromTranslog() throws IOException { translogManager.syncTranslog(); translogManager.close(); translogManager = new InternalTranslogManager( - new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), primaryTerm, globalCheckpoint::get, createTranslogDeletionPolicy(INDEX_SETTINGS), @@ -117,7 +117,7 @@ public void testTranslogRollsGeneration() throws IOException { LocalCheckpointTracker tracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED); try { translogManager = new InternalTranslogManager( - new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), primaryTerm, globalCheckpoint::get, createTranslogDeletionPolicy(INDEX_SETTINGS), @@ -147,7 +147,7 @@ public void testTranslogRollsGeneration() throws IOException { translogManager.syncTranslog(); translogManager.close(); translogManager = new InternalTranslogManager( - new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), primaryTerm, globalCheckpoint::get, createTranslogDeletionPolicy(INDEX_SETTINGS), @@ -182,7 +182,7 @@ public void testTrimOperationsFromTranslog() throws IOException { LocalCheckpointTracker tracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED); try { translogManager = new InternalTranslogManager( - new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), primaryTerm, globalCheckpoint::get, createTranslogDeletionPolicy(INDEX_SETTINGS), @@ -214,7 +214,7 @@ public void testTrimOperationsFromTranslog() throws IOException { translogManager.close(); translogManager = new InternalTranslogManager( - new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), primaryTerm, globalCheckpoint::get, createTranslogDeletionPolicy(INDEX_SETTINGS), @@ -253,7 +253,7 @@ public void testTranslogSync() throws IOException { ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null); AtomicReference translogManagerAtomicReference = new AtomicReference<>(); translogManager = new InternalTranslogManager( - new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), + new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""), primaryTerm, globalCheckpoint::get, createTranslogDeletionPolicy(INDEX_SETTINGS), diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java index dbfc66d6de4b3..4997067b75198 100644 --- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java @@ -291,7 +291,7 @@ private TranslogConfig getTranslogConfig(final Path path, final Settings setting ); final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings); - return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize); + return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize, ""); } private Location addToTranslogAndList(Translog translog, List list, Translog.Operation op) throws IOException { @@ -1452,7 +1452,8 @@ public void testTranslogWriterCanFlushInAddOrReadCall() throws IOException { temp.getTranslogPath(), temp.getIndexSettings(), temp.getBigArrays(), - new ByteSizeValue(1, ByteSizeUnit.KB) + new ByteSizeValue(1, ByteSizeUnit.KB), + "" ); final Set persistedSeqNos = new HashSet<>(); @@ -1550,7 +1551,8 @@ public void testTranslogWriterFsyncedWithLocalTranslog() throws IOException { temp.getTranslogPath(), temp.getIndexSettings(), temp.getBigArrays(), - new ByteSizeValue(1, ByteSizeUnit.KB) + new ByteSizeValue(1, ByteSizeUnit.KB), + "" ); final Set persistedSeqNos = new HashSet<>(); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 233d6f319b797..84506f7ab25ff 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -206,7 +206,7 @@ private TranslogConfig getTranslogConfig(final Path path, final Settings setting ); final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings); - return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize); + return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize, ""); } private BlobStoreRepository createRepository() { @@ -1258,7 +1258,8 @@ public void testTranslogWriterCanFlushInAddOrReadCall() throws IOException { temp.getTranslogPath(), temp.getIndexSettings(), temp.getBigArrays(), - new ByteSizeValue(1, ByteSizeUnit.KB) + new ByteSizeValue(1, ByteSizeUnit.KB), + "" ); final Set persistedSeqNos = new HashSet<>(); @@ -1360,7 +1361,8 @@ public void testTranslogWriterFsyncDisabledInRemoteFsTranslog() throws IOExcepti temp.getTranslogPath(), temp.getIndexSettings(), temp.getBigArrays(), - new ByteSizeValue(1, ByteSizeUnit.KB) + new ByteSizeValue(1, ByteSizeUnit.KB), + "" ); final Set persistedSeqNos = new HashSet<>(); diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java index 43b4d2c9847ab..e17d2770f014a 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java @@ -74,7 +74,7 @@ protected Translog createTranslog(LongSupplier primaryTermSupplier) throws IOExc } protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSupplier) throws IOException { - TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE); + TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""); String translogUUID = Translog.createEmptyTranslog( translogPath, SequenceNumbers.NO_OPS_PERFORMED, diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 6fc4557a75675..a48dbdcdacb71 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -16,6 +16,7 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.common.collect.Tuple; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; @@ -251,8 +252,8 @@ public void testReadMetadataNoFile() throws IOException { assertNoDownloadStats(false); } - // This should happen most of the time - Just a single metadata file - public void testReadMetadataSingleFile() throws IOException { + // This should happen most of the time - + public void testReadMetadataFile() throws IOException { TranslogTransferManager translogTransferManager = new TranslogTransferManager( shardId, transferService, @@ -260,12 +261,16 @@ public void testReadMetadataSingleFile() throws IOException { null, remoteTranslogTransferTracker ); - TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2); - String mdFilename = tm.getFileName(); + TranslogTransferMetadata metadata1 = new TranslogTransferMetadata(1, 1, 1, 2); + String mdFilename1 = metadata1.getFileName(); + + TranslogTransferMetadata metadata2 = new TranslogTransferMetadata(1, 0, 1, 2); + String mdFilename2 = metadata2.getFileName(); doAnswer(invocation -> { LatchedActionListener> latchedActionListener = invocation.getArgument(3); List bmList = new LinkedList<>(); - bmList.add(new PlainBlobMetadata(mdFilename, 1)); + bmList.add(new PlainBlobMetadata(mdFilename1, 1)); + bmList.add(new PlainBlobMetadata(mdFilename2, 1)); latchedActionListener.onResponse(bmList); return null; }).when(transferService) @@ -273,7 +278,7 @@ public void testReadMetadataSingleFile() throws IOException { TranslogTransferMetadata metadata = createTransferSnapshot().getTranslogTransferMetadata(); long delayForMdDownload = 1; - when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenAnswer(invocation -> { + when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename1))).thenAnswer(invocation -> { Thread.sleep(delayForMdDownload); return new ByteArrayInputStream(translogTransferManager.getMetadataBytes(metadata)); }); @@ -496,4 +501,36 @@ private void assertTlogCkpDownloadStats() { // Expect delay for both tlog and ckp file assertTrue(remoteTranslogTransferTracker.getTotalDownloadTimeInMillis() >= 2 * delayForBlobDownload); } + + public void testGetPrimaryTermAndGeneration() { + String tm = new TranslogTransferMetadata(1, 2, 1, 2, "node-1").getFileName(); + assertEquals(new Tuple<>(new Tuple<>(1L, 2L), "node-1"), TranslogTransferMetadata.getNodeIdByPrimaryTermAndGeneration(tm)); + } + + public void testMetadataConflict() throws InterruptedException { + TranslogTransferManager translogTransferManager = new TranslogTransferManager( + shardId, + transferService, + remoteBaseTransferPath, + null, + remoteTranslogTransferTracker + ); + TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2, "node-1"); + String mdFilename = tm.getFileName(); + Thread.sleep(1); + TranslogTransferMetadata tm2 = new TranslogTransferMetadata(1, 1, 1, 2, "node-2"); + String mdFilename2 = tm2.getFileName(); + + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + List bmList = new LinkedList<>(); + bmList.add(new PlainBlobMetadata(mdFilename, 1)); + bmList.add(new PlainBlobMetadata(mdFilename2, 1)); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); + + assertThrows(RuntimeException.class, translogTransferManager::readMetadata); + } } diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index 15f9ee546fe6b..43289a7c89524 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -527,7 +527,7 @@ protected Translog createTranslog(LongSupplier primaryTermSupplier) throws IOExc } protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSupplier) throws IOException { - TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE); + TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""); String translogUUID = Translog.createEmptyTranslog( translogPath, SequenceNumbers.NO_OPS_PERFORMED, @@ -872,7 +872,13 @@ public EngineConfig config( final Engine.EventListener eventListener ) { final IndexWriterConfig iwc = newIndexWriterConfig(); - final TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE); + final TranslogConfig translogConfig = new TranslogConfig( + shardId, + translogPath, + indexSettings, + BigArrays.NON_RECYCLING_INSTANCE, + "" + ); final List extRefreshListenerList = externalRefreshListener == null ? emptyList() : Collections.singletonList(externalRefreshListener); @@ -939,7 +945,7 @@ protected EngineConfig config( .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) .build() ); - TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE); + TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, ""); return new EngineConfig.Builder().shardId(config.getShardId()) .threadPool(config.getThreadPool()) .indexSettings(indexSettings) diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 4505570b8ebe0..466c00d0648dc 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -697,7 +697,8 @@ protected IndexShard newShard( checkpointPublisher, remoteStore, remoteStoreStatsTrackerFactory, - () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, + "dummy-node" ); indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER); if (remoteStoreStatsTrackerFactory != null) { From 699d235a7edad81be19dabbb57e2933ad4f1b6d6 Mon Sep 17 00:00:00 2001 From: Dharmesh Date: Wed, 4 Oct 2023 18:18:41 +0530 Subject: [PATCH 098/388] [Remote Store] Add support to restrict creation & deletion if system repository and mutation of immutable settings of system repository (#9839) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Signed-off-by: Dharmesh 💤 --- CHANGELOG.md | 3 +- .../repositories/azure/AzureRepository.java | 11 +++ .../azure/AzureRepositorySettingsTests.java | 21 ++++ .../gcs/GoogleCloudStorageRepository.java | 11 +++ .../repositories/s3/S3Repository.java | 11 +++ .../repositories/s3/S3RepositoryTests.java | 16 ++++ ...emoteStoreMockRepositoryIntegTestCase.java | 13 +-- .../remotestore/RemoteIndexRecoveryIT.java | 3 +- .../remotestore/RemoteRestoreSnapshotIT.java | 21 +--- .../RemoteStoreBackpressureIT.java | 2 +- .../RemoteStoreBaseIntegTestCase.java | 15 ++- .../RemoteStoreRefreshListenerIT.java | 2 +- .../RemoteStoreRepositoryRegistrationIT.java | 68 +++++++++++++ .../SegmentReplicationUsingRemoteStoreIT.java | 3 +- ...tReplicationWithRemoteStorePressureIT.java | 3 +- .../multipart/RemoteStoreMultipartIT.java | 10 +- .../repositories/RepositoriesServiceIT.java | 12 +++ .../opensearch/snapshots/CloneSnapshotIT.java | 3 - .../snapshots/SystemRepositoryIT.java | 71 ++++++++++++++ .../put/TransportPutRepositoryAction.java | 2 +- .../metadata/RepositoriesMetadata.java | 8 +- .../remotestore/RemoteStoreNodeService.java | 14 ++- .../repositories/RepositoriesService.java | 96 ++++++++++++++++++- .../opensearch/repositories/Repository.java | 11 +++ .../blobstore/BlobStoreRepository.java | 6 ++ .../repositories/fs/FsRepository.java | 10 ++ .../RepositoriesServiceTests.java | 26 +++-- .../repositories/fs/FsRepositoryTests.java | 28 ++++++ .../java/org/opensearch/test/TestCluster.java | 15 ++- 29 files changed, 454 insertions(+), 61 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/snapshots/SystemRepositoryIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a37866012ba1..c847679dafe70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) - Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562)) - Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204)) +- [Remote Store] Add support to restrict creation & deletion if system repository and mutation of immutable settings of system repository ([#9839](https://github.com/opensearch-project/OpenSearch/pull/9839)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) @@ -141,4 +142,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java index 65852c4fc5bd0..381a35bbc11e1 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java @@ -47,6 +47,8 @@ import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.function.Function; @@ -192,4 +194,13 @@ protected ByteSizeValue chunkSize() { public boolean isReadOnly() { return readonly; } + + @Override + public List> getRestrictedSystemRepositorySettings() { + List> restrictedSettings = new ArrayList<>(); + restrictedSettings.addAll(super.getRestrictedSystemRepositorySettings()); + restrictedSettings.add(Repository.BASE_PATH_SETTING); + restrictedSettings.add(Repository.LOCATION_MODE_SETTING); + return restrictedSettings; + } } diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java index c8be30dbaf865..3356e5174592a 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java @@ -34,16 +34,20 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.BlobStoreTestUtil; import org.opensearch.test.OpenSearchTestCase; import org.junit.AfterClass; +import java.util.List; + import reactor.core.scheduler.Schedulers; import static org.hamcrest.Matchers.is; @@ -179,4 +183,21 @@ public void testChunkSize() { ); } + public void testSystemRepositoryDefault() { + assertThat(azureRepository(Settings.EMPTY).isSystemRepository(), is(false)); + } + + public void testSystemRepositoryOn() { + assertThat(azureRepository(Settings.builder().put("system_repository", true).build()).isSystemRepository(), is(true)); + } + + public void testRestrictedSettingsDefault() { + List> restrictedSettings = azureRepository(Settings.EMPTY).getRestrictedSystemRepositorySettings(); + assertThat(restrictedSettings.size(), is(5)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.READONLY_SETTING)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY)); + assertTrue(restrictedSettings.contains(AzureRepository.Repository.BASE_PATH_SETTING)); + assertTrue(restrictedSettings.contains(AzureRepository.Repository.LOCATION_MODE_SETTING)); + } } diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java index c42cd1802f6e9..13671bc2aa8d6 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -46,6 +46,8 @@ import org.opensearch.repositories.RepositoryException; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -138,6 +140,15 @@ protected ByteSizeValue chunkSize() { return chunkSize; } + @Override + public List> getRestrictedSystemRepositorySettings() { + List> restrictedSettings = new ArrayList<>(); + restrictedSettings.addAll(super.getRestrictedSystemRepositorySettings()); + restrictedSettings.add(BUCKET); + restrictedSettings.add(BASE_PATH); + return restrictedSettings; + } + /** * Get a given setting from the repository settings, throwing a {@link RepositoryException} if the setting does not exist or is empty. */ diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java index 59111e94df22e..8e69fc39f128e 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java @@ -62,7 +62,9 @@ import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.threadpool.Scheduler; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -388,6 +390,15 @@ protected ByteSizeValue chunkSize() { return chunkSize; } + @Override + public List> getRestrictedSystemRepositorySettings() { + List> restrictedSettings = new ArrayList<>(); + restrictedSettings.addAll(super.getRestrictedSystemRepositorySettings()); + restrictedSettings.add(BUCKET_SETTING); + restrictedSettings.add(BASE_PATH_SETTING); + return restrictedSettings; + } + @Override protected void doClose() { final Scheduler.Cancellable cancellable = finalizationFuture.getAndSet(null); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java index 533c3aa17009d..9ea8d98505161 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java @@ -36,17 +36,20 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.repositories.RepositoryException; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.BlobStoreTestUtil; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; import java.nio.file.Path; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -133,6 +136,19 @@ public void testDefaultBufferSize() { } } + public void testRestrictedSettingsDefault() { + final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", Settings.EMPTY); + try (S3Repository s3repo = createS3Repo(metadata)) { + List> restrictedSettings = s3repo.getRestrictedSystemRepositorySettings(); + assertThat(restrictedSettings.size(), is(5)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.READONLY_SETTING)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY)); + assertTrue(restrictedSettings.contains(S3Repository.BUCKET_SETTING)); + assertTrue(restrictedSettings.contains(S3Repository.BASE_PATH_SETTING)); + } + } + private S3Repository createS3Repo(RepositoryMetadata metadata) { return new S3Repository( metadata, diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index bc55f6cc2cbcb..9d4d8aa24bd51 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -33,7 +33,6 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { @@ -107,11 +106,11 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu .build(); } - protected void deleteRepo() { - logger.info("--> Deleting the repository={}", REPOSITORY_NAME); - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); - logger.info("--> Deleting the repository={}", TRANSLOG_REPOSITORY_NAME); - assertAcked(clusterAdmin().prepareDeleteRepository(TRANSLOG_REPOSITORY_NAME)); + protected void cleanupRepo() { + logger.info("--> Cleanup the repository={}", REPOSITORY_NAME); + clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).execute().actionGet(); + logger.info("--> Cleanup the repository={}", TRANSLOG_REPOSITORY_NAME); + clusterAdmin().prepareCleanupRepository(TRANSLOG_REPOSITORY_NAME).execute().actionGet(); } protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { @@ -125,6 +124,8 @@ protected String setup(Path repoLocation, double ioFailureRate, String skipExcep settings.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT); } + disableRepoConsistencyCheck("Remote Store Creates System Repository"); + internalCluster().startClusterManagerOnlyNode(settings.build()); String dataNodeName = internalCluster().startDataOnlyNode(settings.build()); createIndex(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java index 4eb1cc7703735..c957f1b338bfe 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -23,7 +23,6 @@ import java.nio.file.Path; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteIndexRecoveryIT extends IndexRecoveryIT { @@ -57,7 +56,7 @@ public Settings indexSettings() { @After public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get(); } @Override diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 4ebccb9b9e551..865b2d13f189e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -57,7 +57,7 @@ public void setup() { @After public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository(BASE_REMOTE_REPO)); + clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get(); } @Override @@ -422,7 +422,7 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); } - public void testRestoreShallowSnapshotRepositoryOverriden() throws ExecutionException, InterruptedException { + public void testRestoreShallowSnapshotRepository() throws ExecutionException, InterruptedException { String indexName1 = "testindex1"; String snapshotRepoName = "test-restore-snapshot-repo"; String remoteStoreRepoNameUpdated = "test-rs-repo-updated" + TEST_REMOTE_STORE_REPO_SUFFIX; @@ -464,22 +464,7 @@ public void testRestoreShallowSnapshotRepositoryOverriden() throws ExecutionExce assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); - createRepository(BASE_REMOTE_REPO, "fs", absolutePath2); - - RestoreSnapshotResponse restoreSnapshotResponse = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1) - .setRenamePattern(indexName1) - .setRenameReplacement(restoredIndexName1) - .get(); - - assertTrue(restoreSnapshotResponse.getRestoreInfo().failedShards() > 0); - - ensureRed(restoredIndexName1); - - client().admin().indices().close(Requests.closeIndexRequest(restoredIndexName1)).get(); + client().admin().indices().close(Requests.closeIndexRequest(indexName1)).get(); createRepository(remoteStoreRepoNameUpdated, "fs", remoteRepoPath); RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() .cluster() diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java index d02c5bf54fbed..3462054c23630 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java @@ -112,7 +112,7 @@ private void validateBackpressure( stats = stats(); indexDocAndRefresh(initialSource, initialDocsToIndex); assertEquals(rejectionCount, stats.rejectionCount); - deleteRepo(); + cleanupRepo(); } private RemoteSegmentTransferTracker.Stats stats() { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 157f8e41fee24..bf536557b1485 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -27,6 +27,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; @@ -50,7 +51,6 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String REPOSITORY_NAME = "test-remote-store-repo"; @@ -314,8 +314,8 @@ public void teardown() { clusterSettingsSuppliedByTest = false; assertRemoteStoreRepositoryOnAllNodes(REPOSITORY_NAME); assertRemoteStoreRepositoryOnAllNodes(REPOSITORY_2_NAME); - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); + clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get(); + clusterAdmin().prepareCleanupRepository(REPOSITORY_2_NAME).get(); } public RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { @@ -343,11 +343,18 @@ public void assertRemoteStoreRepositoryOnAllNodes(String repositoryName) { .custom(RepositoriesMetadata.TYPE); RepositoryMetadata actualRepository = repositories.repository(repositoryName); + final RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); + final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repositoryName); + for (String nodeName : internalCluster().getNodeNames()) { ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); DiscoveryNode node = clusterService.localNode(); RepositoryMetadata expectedRepository = buildRepositoryMetadata(node, repositoryName); - assertTrue(actualRepository.equalsIgnoreGenerations(expectedRepository)); + + // Validated that all the restricted settings are entact on all the nodes. + repository.getRestrictedSystemRepositorySettings() + .stream() + .forEach(setting -> assertEquals(setting.get(actualRepository.settings()), setting.get(expectedRepository.settings()))); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java index b97e93f323fb2..88760b7bbfad2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java @@ -56,7 +56,7 @@ public void testRemoteRefreshRetryOnFailure() throws Exception { logger.info("Local files = {}, Repo files = {}", sortedFilesInLocal, sortedFilesInRepo); assertTrue(filesInRepo.containsAll(filesInLocal)); }, 90, TimeUnit.SECONDS); - deleteRepo(); + cleanupRepo(); } public void testRemoteRefreshSegmentPressureSettingChanged() { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 4d56a1e94e3fc..002a149f0c286 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -8,6 +8,10 @@ package org.opensearch.remotestore; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.disruption.NetworkDisruption; @@ -94,4 +98,68 @@ public void testMultiNodeClusterRandomNodeRecoverNetworkIsolation() { internalCluster().clearDisruptionScheme(); } + + public void testMultiNodeClusterRandomNodeRecoverNetworkIsolationPostNonRestrictedSettingsUpdate() { + Set nodesInOneSide = internalCluster().startNodes(3).stream().collect(Collectors.toCollection(HashSet::new)); + Set nodesInAnotherSide = internalCluster().startNodes(3).stream().collect(Collectors.toCollection(HashSet::new)); + ensureStableCluster(6); + + NetworkDisruption networkDisruption = new NetworkDisruption( + new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInAnotherSide), + NetworkDisruption.DISCONNECT + ); + internalCluster().setDisruptionScheme(networkDisruption); + + networkDisruption.startDisrupting(); + + final Client client = client(nodesInOneSide.iterator().next()); + RepositoryMetadata repositoryMetadata = client.admin() + .cluster() + .prepareGetRepositories(REPOSITORY_NAME) + .get() + .repositories() + .get(0); + Settings.Builder updatedSettings = Settings.builder().put(repositoryMetadata.settings()).put("chunk_size", new ByteSizeValue(20)); + updatedSettings.remove("system_repository"); + + client.admin() + .cluster() + .preparePutRepository(repositoryMetadata.name()) + .setType(repositoryMetadata.type()) + .setSettings(updatedSettings) + .get(); + + ensureStableCluster(3, nodesInOneSide.stream().findAny().get()); + networkDisruption.stopDisrupting(); + + ensureStableCluster(6); + + internalCluster().clearDisruptionScheme(); + } + + public void testNodeRestartPostNonRestrictedSettingsUpdate() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startNodes(3); + + final Client client = client(); + RepositoryMetadata repositoryMetadata = client.admin() + .cluster() + .prepareGetRepositories(REPOSITORY_NAME) + .get() + .repositories() + .get(0); + Settings.Builder updatedSettings = Settings.builder().put(repositoryMetadata.settings()).put("chunk_size", new ByteSizeValue(20)); + updatedSettings.remove("system_repository"); + + client.admin() + .cluster() + .preparePutRepository(repositoryMetadata.name()) + .setType(repositoryMetadata.type()) + .setSettings(updatedSettings) + .get(); + + internalCluster().restartRandomDataNode(); + + ensureStableCluster(4); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index 45c3ef7f5bae5..23864c35ad154 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -17,7 +17,6 @@ import java.nio.file.Path; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; /** * This class runs Segment Replication Integ test suite with remote store enabled. @@ -50,6 +49,6 @@ public void setup() { @After public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get(); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java index 0da4d81a8871e..6cfc76b7e3223 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java @@ -17,7 +17,6 @@ import java.nio.file.Path; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; /** * This class executes the SegmentReplicationPressureIT suite with remote store integration enabled. @@ -49,6 +48,6 @@ public void setup() { @After public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get(); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java index 98fab139f4902..3dfde6f472525 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -108,8 +108,15 @@ public RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String nam } public void testRateLimitedRemoteUploads() throws Exception { + clusterSettingsSuppliedByTest = true; overrideBuildRepositoryMetadata = true; - internalCluster().startNode(); + Settings.Builder clusterSettings = Settings.builder() + .put(remoteStoreClusterSettings(REPOSITORY_NAME, repositoryLocation, REPOSITORY_2_NAME, repositoryLocation)); + clusterSettings.put( + String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, REPOSITORY_NAME), + MockFsRepositoryPlugin.TYPE + ); + internalCluster().startNode(clusterSettings.build()); Client client = client(); logger.info("--> updating repository"); assertAcked( @@ -119,7 +126,6 @@ public void testRateLimitedRemoteUploads() throws Exception { .setType(MockFsRepositoryPlugin.TYPE) .setSettings( Settings.builder() - .put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true) .put("location", repositoryLocation) .put("compress", compress) .put("max_remote_upload_bytes_per_sec", "1kb") diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java index f149d538cc47a..b8415f4b41815 100644 --- a/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java @@ -108,4 +108,16 @@ public void testUpdateRepository() { final Repository updatedRepository = repositoriesService.repository(repositoryName); assertThat(updatedRepository, updated ? not(sameInstance(originalRepository)) : sameInstance(originalRepository)); } + + public void testSystemRepositoryCantBeCreated() { + internalCluster(); + final String repositoryName = "test-repo"; + final Client client = client(); + final Settings.Builder repoSettings = Settings.builder().put("system_repository", true).put("location", randomRepoPath()); + + assertThrows( + RepositoryException.class, + () -> client.admin().cluster().preparePutRepository(repositoryName).setType(FsRepository.TYPE).setSettings(repoSettings).get() + ); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 066d82483ae91..83f93ab9ff6b5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -215,9 +215,6 @@ public void testShallowCloneNameAvailability() throws Exception { final Path shallowSnapshotRepoPath = randomRepoPath(); createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SystemRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SystemRepositoryIT.java new file mode 100644 index 0000000000000..f50fc691fb232 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SystemRepositoryIT.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.repositories.RepositoryException; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.Before; + +import java.nio.file.Path; + +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class SystemRepositoryIT extends AbstractSnapshotIntegTestCase { + protected Path absolutePath; + final String systemRepoName = "system-repo-name"; + + @Before + public void setup() { + absolutePath = randomRepoPath().toAbsolutePath(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(systemRepoName, absolutePath)) + .build(); + } + + public void testRestrictedSettingsCantBeUpdated() { + disableRepoConsistencyCheck("System repository is being used for the test"); + + internalCluster().startNode(); + final Client client = client(); + final Settings.Builder repoSettings = Settings.builder().put("location", randomRepoPath()); + + RepositoryException e = expectThrows( + RepositoryException.class, + () -> client.admin().cluster().preparePutRepository(systemRepoName).setType("mock").setSettings(repoSettings).get() + ); + assertEquals( + e.getMessage(), + "[system-repo-name] trying to modify an unmodifiable attribute type of system " + + "repository from current value [fs] to new value [mock]" + ); + } + + public void testSystemRepositoryNonRestrictedSettingsCanBeUpdated() { + disableRepoConsistencyCheck("System repository is being used for the test"); + + internalCluster().startNode(); + final Client client = client(); + final Settings.Builder repoSettings = Settings.builder().put("location", absolutePath).put("chunk_size", new ByteSizeValue(20)); + + assertAcked( + client.admin().cluster().preparePutRepository(systemRepoName).setType(FsRepository.TYPE).setSettings(repoSettings).get() + ); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java index 644f23d2bafe6..1eadab6b1352e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java @@ -100,7 +100,7 @@ protected void clusterManagerOperation( ClusterState state, final ActionListener listener ) { - repositoriesService.registerRepository( + repositoriesService.registerOrUpdateRepository( request, ActionListener.delegateFailure( listener, diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 1ab402e1bde4e..a5ef337c3b62a 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -54,6 +54,8 @@ import java.util.EnumSet; import java.util.List; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; + /** * Contains metadata about registered snapshot repositories * @@ -288,8 +290,12 @@ public static void toXContent(RepositoryMetadata repository, XContentBuilder bui if (repository.cryptoMetadata() != null) { repository.cryptoMetadata().toXContent(repository.cryptoMetadata(), builder, params); } + Settings settings = repository.settings(); + if (SYSTEM_REPOSITORY_SETTING.get(settings)) { + settings = repository.settings().filter(s -> !s.equals(SYSTEM_REPOSITORY_SETTING.getKey())); + } builder.startObject("settings"); - repository.settings().toXContent(builder, params); + settings.toXContent(builder, params); builder.endObject(); if (params.paramAsBoolean(HIDE_GENERATIONS_PARAM, false) == false) { diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 26c078353d12a..ca2413a057a6b 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -17,6 +17,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryException; import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; @@ -133,10 +134,19 @@ public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode boolean repositoryAlreadyPresent = false; for (RepositoryMetadata existingRepositoryMetadata : existingRepositories.repositories()) { if (newRepositoryMetadata.name().equals(existingRepositoryMetadata.name())) { - if (newRepositoryMetadata.equalsIgnoreGenerations(existingRepositoryMetadata)) { + try { + // This will help in handling two scenarios - + // 1. When a fresh cluster is formed and a node tries to join the cluster, the repository + // metadata constructed from the node attributes of the joining node will be validated + // against the repository information provided by existing nodes in cluster state. + // 2. It's possible to update repository settings except the restricted ones post the + // creation of a system repository and if a node drops we will need to allow it to join + // even if the non-restricted system repository settings are now different. + repositoriesService.get().ensureValidSystemRepositoryUpdate(newRepositoryMetadata, existingRepositoryMetadata); + newRepositoryMetadata = existingRepositoryMetadata; repositoryAlreadyPresent = true; break; - } else { + } catch (RepositoryException e) { throw new IllegalStateException( "new repository metadata [" + newRepositoryMetadata diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 3d6679b3ef80e..0361a71116a16 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -84,6 +84,7 @@ import java.util.stream.Stream; import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; /** * Service responsible for maintaining and providing access to snapshot repositories on nodes. @@ -154,7 +155,7 @@ public RepositoriesService( } /** - * Registers new repository in the cluster + * Registers new repository or updates an existing repository in the cluster *

* This method can be only called on the cluster-manager node. It tries to create a new repository on the master * and if it was successful it adds new repository to cluster metadata. @@ -162,7 +163,7 @@ public RepositoriesService( * @param request register repository request * @param listener register repository listener */ - public void registerRepository(final PutRepositoryRequest request, final ActionListener listener) { + public void registerOrUpdateRepository(final PutRepositoryRequest request, final ActionListener listener) { assert lifecycle.started() : "Trying to register new repository but service is in state [" + lifecycle.state() + "]"; final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata( @@ -236,14 +237,30 @@ public ClusterState execute(ClusterState currentState) { List repositoriesMetadata = new ArrayList<>(repositories.repositories().size() + 1); for (RepositoryMetadata repositoryMetadata : repositories.repositories()) { - if (repositoryMetadata.name().equals(newRepositoryMetadata.name())) { - if (newRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)) { + RepositoryMetadata updatedRepositoryMetadata = newRepositoryMetadata; + if (isSystemRepositorySettingPresent(repositoryMetadata.settings())) { + Settings updatedSettings = Settings.builder() + .put(newRepositoryMetadata.settings()) + .put(SYSTEM_REPOSITORY_SETTING.getKey(), true) + .build(); + updatedRepositoryMetadata = new RepositoryMetadata( + newRepositoryMetadata.name(), + newRepositoryMetadata.type(), + updatedSettings, + newRepositoryMetadata.cryptoMetadata() + ); + } + if (repositoryMetadata.name().equals(updatedRepositoryMetadata.name())) { + if (updatedRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)) { // Previous version is the same as this one no update is needed. return currentState; } ensureCryptoSettingsAreSame(repositoryMetadata, request); found = true; - repositoriesMetadata.add(newRepositoryMetadata); + if (isSystemRepositorySettingPresent(repositoryMetadata.settings())) { + ensureValidSystemRepositoryUpdate(updatedRepositoryMetadata, repositoryMetadata); + } + repositoriesMetadata.add(updatedRepositoryMetadata); } else { repositoriesMetadata.add(repositoryMetadata); } @@ -315,6 +332,7 @@ public ClusterState execute(ClusterState currentState) { for (RepositoryMetadata repositoryMetadata : repositories.repositories()) { if (Regex.simpleMatch(request.name(), repositoryMetadata.name())) { ensureRepositoryNotInUse(currentState, repositoryMetadata.name()); + ensureNotSystemRepository(repositoryMetadata); logger.info("delete repository [{}]", repositoryMetadata.name()); changed = true; } else { @@ -682,6 +700,15 @@ public static void validateRepositoryMetadataSettings( + minVersionInCluster ); } + // Validation to not allow users to create system repository via put repository call. + if (isSystemRepositorySettingPresent(repositoryMetadataSettings)) { + throw new RepositoryException( + repositoryName, + "setting " + + SYSTEM_REPOSITORY_SETTING.getKey() + + " cannot provide system repository setting; this setting is managed by OpenSearch" + ); + } } private static void ensureRepositoryNotInUse(ClusterState clusterState, String repository) { @@ -756,6 +783,65 @@ public void updateRepositoriesMap(Map repos) { } } + private static void ensureNotSystemRepository(RepositoryMetadata repositoryMetadata) { + if (isSystemRepositorySettingPresent(repositoryMetadata.settings())) { + throw new RepositoryException(repositoryMetadata.name(), "cannot delete a system repository"); + } + } + + private static boolean isSystemRepositorySettingPresent(Settings repositoryMetadataSettings) { + return SYSTEM_REPOSITORY_SETTING.get(repositoryMetadataSettings); + } + + private static boolean isValueEqual(String key, String newValue, String currentValue) { + if (newValue == null && currentValue == null) { + return true; + } + if (newValue == null) { + throw new IllegalArgumentException("[" + key + "] cannot be empty, " + "current value [" + currentValue + "]"); + } + if (newValue.equals(currentValue) == false) { + throw new IllegalArgumentException( + "trying to modify an unmodifiable attribute " + + key + + " of system repository from " + + "current value [" + + currentValue + + "] to new value [" + + newValue + + "]" + ); + } + return true; + } + + public void ensureValidSystemRepositoryUpdate(RepositoryMetadata newRepositoryMetadata, RepositoryMetadata currentRepositoryMetadata) { + if (isSystemRepositorySettingPresent(currentRepositoryMetadata.settings())) { + try { + isValueEqual("type", newRepositoryMetadata.type(), currentRepositoryMetadata.type()); + + Repository repository = repositories.get(currentRepositoryMetadata.name()); + Settings newRepositoryMetadataSettings = newRepositoryMetadata.settings(); + Settings currentRepositoryMetadataSettings = currentRepositoryMetadata.settings(); + + List restrictedSettings = repository.getRestrictedSystemRepositorySettings() + .stream() + .map(setting -> setting.getKey()) + .collect(Collectors.toList()); + + for (String restrictedSettingKey : restrictedSettings) { + isValueEqual( + restrictedSettingKey, + newRepositoryMetadataSettings.get(restrictedSettingKey), + currentRepositoryMetadataSettings.get(restrictedSettingKey) + ); + } + } catch (IllegalArgumentException e) { + throw new RepositoryException(currentRepositoryMetadata.name(), e.getMessage()); + } + } + } + @Override protected void doStart() { diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 10f3dc2b6b340..02ba60cb23c4d 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -42,6 +42,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; import org.opensearch.common.lifecycle.LifecycleComponent; +import org.opensearch.common.settings.Setting; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.mapper.MapperService; @@ -55,6 +56,8 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; @@ -342,6 +345,14 @@ void restoreShard( ActionListener listener ); + /** + * Returns the list of restricted system repository settings that cannot be mutated post repository creation. + * @return list of settings + */ + default List> getRestrictedSystemRepositorySettings() { + return Collections.emptyList(); + } + /** * Returns Snapshot Shard Metadata for remote store interop enabled snapshot. *

diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 490ebda24bf60..3481e43cf4c72 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -149,6 +149,7 @@ import java.io.InputStream; import java.nio.file.NoSuchFileException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -3138,6 +3139,11 @@ public InputStream maybeRateLimitSnapshots(InputStream stream) { return maybeRateLimit(stream, () -> snapshotRateLimiter, snapshotRateLimitingTimeInNanos, BlobStoreTransferContext.SNAPSHOT); } + @Override + public List> getRestrictedSystemRepositorySettings() { + return Arrays.asList(SYSTEM_REPOSITORY_SETTING, READONLY_SETTING, REMOTE_STORE_INDEX_SHALLOW_COPY); + } + @Override public RemoteStoreShardShallowCopySnapshot getRemoteStoreShallowCopyShardMetadata( SnapshotId snapshotId, diff --git a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java index 3009466f03635..d432bab93bd71 100644 --- a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java @@ -50,6 +50,8 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.function.Function; /** @@ -187,4 +189,12 @@ protected ByteSizeValue chunkSize() { public BlobPath basePath() { return basePath; } + + @Override + public List> getRestrictedSystemRepositorySettings() { + List> restrictedSettings = new ArrayList<>(); + restrictedSettings.addAll(super.getRestrictedSystemRepositorySettings()); + restrictedSettings.add(LOCATION_SETTING); + return restrictedSettings; + } } diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index 889d0dc6ddb14..cc4f4f7b6413d 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -94,6 +94,7 @@ import java.util.function.Consumer; import java.util.function.Function; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -201,6 +202,13 @@ public void testRegisterRejectsInvalidRepositoryNames() { } } + public void testUpdateOrRegisterRejectsForSystemRepository() { + String repoName = "name"; + PutRepositoryRequest request = new PutRepositoryRequest(repoName); + request.settings(Settings.builder().put(SYSTEM_REPOSITORY_SETTING.getKey(), true).build()); + expectThrows(RepositoryException.class, () -> repositoriesService.registerOrUpdateRepository(request, null)); + } + public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() { String repoName = "name"; expectThrows(RepositoryMissingException.class, () -> repositoriesService.repository(repoName)); @@ -310,22 +318,22 @@ public void testRepositoryUpdateWithDifferentCryptoMetadata() { assertNotNull(repository.cryptoHandler); assertEquals(kpTypeA, repository.cryptoHandler.kpType); - expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); + expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerOrUpdateRepository(request, null)); CryptoSettings cryptoSettings = new CryptoSettings(keyProviderName); cryptoSettings.keyProviderType(kpTypeA); cryptoSettings.settings(Settings.builder().put("key-1", "val-1")); request.cryptoSettings(cryptoSettings); - expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); + expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerOrUpdateRepository(request, null)); cryptoSettings.settings(Settings.builder()); cryptoSettings.keyProviderName("random"); - expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerRepository(request, null)); + expectThrows(IllegalArgumentException.class, () -> repositoriesService.registerOrUpdateRepository(request, null)); cryptoSettings.keyProviderName(keyProviderName); assertEquals(kpTypeA, repository.cryptoHandler.kpType); - repositoriesService.registerRepository(request, null); + repositoriesService.registerOrUpdateRepository(request, null); } public void testCryptoManagerClusterStateChanges() { @@ -355,7 +363,7 @@ public void testCryptoManagerClusterStateChanges() { verified, repositoryMetadata ); - repositoriesService.registerRepository(request, null); + repositoriesService.registerOrUpdateRepository(request, null); MeteredRepositoryTypeA repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); assertNotNull(repository.cryptoHandler); assertEquals(kpTypeA, repository.cryptoHandler.kpType); @@ -375,7 +383,7 @@ public void testCryptoManagerClusterStateChanges() { verified, repositoryMetadata ); - repositoriesService.registerRepository(request, null); + repositoriesService.registerOrUpdateRepository(request, null); repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); assertNotNull(repository.cryptoHandler); @@ -397,7 +405,7 @@ public void testCryptoManagerClusterStateChanges() { verified, repositoryMetadata ); - repositoriesService.registerRepository(request, null); + repositoriesService.registerOrUpdateRepository(request, null); repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); assertNotNull(repository.cryptoHandler); assertEquals(kpTypeA, repository.cryptoHandler.kpType); @@ -418,7 +426,7 @@ public void testCryptoManagerClusterStateChanges() { verified, repositoryMetadata ); - repositoriesService.registerRepository(request, null); + repositoriesService.registerOrUpdateRepository(request, null); repository = (MeteredRepositoryTypeA) repositoriesService.repository(repoName); assertNotNull(repository.cryptoHandler); assertEquals(kpTypeB, repository.cryptoHandler.kpType); @@ -530,7 +538,7 @@ private ClusterState emptyState() { private void assertThrowsOnRegister(String repoName) { PutRepositoryRequest request = new PutRepositoryRequest(repoName); - expectThrows(RepositoryException.class, () -> repositoriesService.registerRepository(request, null)); + expectThrows(RepositoryException.class, () -> repositoriesService.registerOrUpdateRepository(request, null)); } private static class TestCryptoProvider implements CryptoHandler { diff --git a/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java index 303f60283f69f..d9f599714805b 100644 --- a/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java @@ -58,6 +58,7 @@ import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.index.shard.ShardId; @@ -70,6 +71,7 @@ import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.repositories.IndexId; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.BlobStoreTestUtil; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; @@ -90,6 +92,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.is; public class FsRepositoryTests extends OpenSearchTestCase { @@ -218,6 +221,31 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { } } + public void testRestrictedSettingsDefault() { + Path repo = createTempDir(); + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()) + .put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()) + .put("location", repo) + .put(FsRepository.BASE_PATH_SETTING.getKey(), "my_base_path") + .build(); + RepositoryMetadata metadata = new RepositoryMetadata("test", "fs", settings); + FsRepository repository = new FsRepository( + metadata, + new Environment(settings, null), + NamedXContentRegistry.EMPTY, + BlobStoreTestUtil.mockClusterService(), + new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + + List> restrictedSettings = repository.getRestrictedSystemRepositorySettings(); + assertThat(restrictedSettings.size(), is(4)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.READONLY_SETTING)); + assertTrue(restrictedSettings.contains(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY)); + assertTrue(restrictedSettings.contains(FsRepository.LOCATION_SETTING)); + } + private void runGeneric(ThreadPool threadPool, Runnable runnable) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); threadPool.generic().submit(() -> { diff --git a/test/framework/src/main/java/org/opensearch/test/TestCluster.java b/test/framework/src/main/java/org/opensearch/test/TestCluster.java index 61742cd4fb827..a9d9abf69ee41 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/TestCluster.java @@ -42,10 +42,12 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.index.IndexNotFoundException; import org.opensearch.indices.IndexTemplateMissingException; import org.opensearch.repositories.RepositoryMissingException; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.io.Closeable; @@ -253,7 +255,18 @@ public void wipeRepositories(String... repositories) { } for (String repository : repositories) { try { - client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet(); + List repositoryMetadata = client().admin() + .cluster() + .prepareGetRepositories(repository) + .execute() + .actionGet() + .repositories(); + if (repositoryMetadata.isEmpty() == false + && BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.get(repositoryMetadata.get(0).settings()) == true) { + client().admin().cluster().prepareCleanupRepository(repository).execute().actionGet(); + } else { + client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet(); + } } catch (RepositoryMissingException ex) { // ignore } From 677ff7513598d5622285a31bd1dd7011abd45bf7 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 4 Oct 2023 18:47:00 +0530 Subject: [PATCH 099/388] Retry on failure to acquire lock on remote metadata (#10341) Signed-off-by: Sachin Kale --- .../snapshots/SnapshotShardsService.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java index af2f925f89726..1c25d8c71f948 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.IndexCommit; import org.opensearch.Version; +import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterStateListener; import org.opensearch.cluster.SnapshotsInProgress; @@ -73,6 +74,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.nio.file.NoSuchFileException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -401,18 +403,32 @@ private void snapshot( try { if (remoteStoreIndexShallowCopy && indexShard.indexSettings().isRemoteStoreEnabled()) { long startTime = threadPool.relativeTimeInMillis(); + long primaryTerm = indexShard.getOperationPrimaryTerm(); // we flush first to make sure we get the latest writes snapshotted wrappedSnapshot = indexShard.acquireLastIndexCommitAndRefresh(true); - long primaryTerm = indexShard.getOperationPrimaryTerm(); - final IndexCommit snapshotIndexCommit = wrappedSnapshot.get(); + IndexCommit snapshotIndexCommit = wrappedSnapshot.get(); long commitGeneration = snapshotIndexCommit.getGeneration(); - indexShard.acquireLockOnCommitData(snapshot.getSnapshotId().getUUID(), primaryTerm, commitGeneration); + try { + indexShard.acquireLockOnCommitData(snapshot.getSnapshotId().getUUID(), primaryTerm, commitGeneration); + } catch (NoSuchFileException e) { + wrappedSnapshot.close(); + logger.warn( + "Exception while acquiring lock on primaryTerm = {} and generation = {}", + primaryTerm, + commitGeneration + ); + indexShard.flush(new FlushRequest(shardId.getIndexName()).force(true)); + wrappedSnapshot = indexShard.acquireLastIndexCommit(false); + snapshotIndexCommit = wrappedSnapshot.get(); + commitGeneration = snapshotIndexCommit.getGeneration(); + indexShard.acquireLockOnCommitData(snapshot.getSnapshotId().getUUID(), primaryTerm, commitGeneration); + } try { repository.snapshotRemoteStoreIndexShard( indexShard.store(), snapshot.getSnapshotId(), indexId, - wrappedSnapshot.get(), + snapshotIndexCommit, getShardStateId(indexShard, snapshotIndexCommit), snapshotStatus, primaryTerm, From e5024a834d090703bae97c0d989ad61baf1c4889 Mon Sep 17 00:00:00 2001 From: Ashish Date: Wed, 4 Oct 2023 22:45:47 +0530 Subject: [PATCH 100/388] Enable remote segment upload backpressure by default (#10356) Signed-off-by: Ashish Singh --- CHANGELOG.md | 1 + .../opensearch/index/remote/RemoteStorePressureSettings.java | 2 +- .../index/remote/RemoteStorePressureServiceTests.java | 2 +- .../index/remote/RemoteStorePressureSettingsTests.java | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c847679dafe70..f3a18154b023a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,6 +125,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122)) - Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246)) - Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200)) +- Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureSettings.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureSettings.java index 864fe24c282a2..e66aa3444c214 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureSettings.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureSettings.java @@ -30,7 +30,7 @@ static class Defaults { public static final Setting REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED = Setting.boolSetting( "remote_store.segment.pressure.enabled", - false, + true, Setting.Property.Dynamic, Setting.Property.NodeScope ); diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java index ada8d9983aa3d..de610083f3327 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java @@ -58,7 +58,7 @@ public void tearDown() throws Exception { public void testIsSegmentsUploadBackpressureEnabled() { remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, Settings.EMPTY); pressureService = new RemoteStorePressureService(clusterService, Settings.EMPTY, remoteStoreStatsTrackerFactory); - assertFalse(pressureService.isSegmentsUploadBackpressureEnabled()); + assertTrue(pressureService.isSegmentsUploadBackpressureEnabled()); Settings newSettings = Settings.builder() .put(RemoteStorePressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey(), "true") diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java index f5514b8936a2f..064c6c10eba02 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureSettingsTests.java @@ -48,7 +48,7 @@ public void testGetDefaultSettings() { ); // Check remote refresh segment pressure enabled is false - assertFalse(pressureSettings.isRemoteRefreshSegmentPressureEnabled()); + assertTrue(pressureSettings.isRemoteRefreshSegmentPressureEnabled()); // Check bytes lag variance threshold default value assertEquals(10.0, pressureSettings.getBytesLagVarianceFactor(), 0.0d); From a0cb34442f704309173e127675cc4ae83c7fb197 Mon Sep 17 00:00:00 2001 From: Ashish Date: Thu, 5 Oct 2023 00:34:03 +0530 Subject: [PATCH 101/388] Fix shard failure due to translog tragic close on upload failure (#10363) Signed-off-by: Ashish Singh --- .../index/translog/RemoteFsTranslog.java | 23 +++++-------------- .../index/translog/RemoteFsTranslogTests.java | 23 ++++++++++++++++++- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 857d90e2e2ac2..29c825fd383c5 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -242,15 +242,10 @@ public static TranslogTransferManager buildTranslogTransferManager( @Override public boolean ensureSynced(Location location) throws IOException { - try { - assert location.generation <= current.getGeneration(); - if (location.generation == current.getGeneration()) { - ensureOpen(); - return prepareAndUpload(primaryTermSupplier.getAsLong(), location.generation); - } - } catch (final Exception ex) { - closeOnTragicEvent(ex); - throw ex; + assert location.generation <= current.getGeneration(); + if (location.generation == current.getGeneration()) { + ensureOpen(); + return prepareAndUpload(primaryTermSupplier.getAsLong(), location.generation); } return false; } @@ -355,14 +350,8 @@ private boolean syncToDisk() throws IOException { @Override public void sync() throws IOException { - try { - if (syncToDisk() || syncNeeded()) { - prepareAndUpload(primaryTermSupplier.getAsLong(), null); - } - } catch (final Exception e) { - tragedy.setTragicException(e); - closeOnTragicEvent(e); - throw e; + if (syncToDisk() || syncNeeded()) { + prepareAndUpload(primaryTermSupplier.getAsLong(), null); } } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 84506f7ab25ff..b2310010620f7 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -67,6 +67,7 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -1049,7 +1050,7 @@ public void testSyncUpTo() throws IOException { } } - public void testSyncUpFailure() throws IOException { + public void testSyncUpLocationFailure() throws IOException { int translogOperations = randomIntBetween(1, 20); int count = 0; fail.failAlways(); @@ -1101,6 +1102,26 @@ public void testSyncUpFailure() throws IOException { assertDownloadStatsNoDownloads(statsTracker); } + public void testSyncUpAlwaysFailure() throws IOException { + int translogOperations = randomIntBetween(1, 20); + int count = 0; + fail.failAlways(); + for (int op = 0; op < translogOperations; op++) { + translog.add( + new Translog.Index(String.valueOf(op), count, primaryTerm.get(), Integer.toString(count).getBytes(StandardCharsets.UTF_8)) + ); + try { + translog.sync(); + fail("io exception expected"); + } catch (IOException e) { + assertTrue("at least one operation pending", translog.syncNeeded()); + } + } + assertTrue(translog.isOpen()); + fail.failNever(); + translog.sync(); + } + public void testSyncUpToStream() throws IOException { int iters = randomIntBetween(5, 10); for (int i = 0; i < iters; i++) { From 3ed7cc43f0d7c0cb69b07c4e294b7d43aed4b3a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 15:25:23 -0400 Subject: [PATCH 102/388] Bump org.xerial.snappy:snappy-java from 1.1.10.4 to 1.1.10.5 in /test/fixtures/hdfs-fixture (#10299) * Bump org.xerial.snappy:snappy-java in /test/fixtures/hdfs-fixture Bumps [org.xerial.snappy:snappy-java](https://github.com/xerial/snappy-java) from 1.1.10.4 to 1.1.10.5. - [Release notes](https://github.com/xerial/snappy-java/releases) - [Commits](https://github.com/xerial/snappy-java/compare/v1.1.10.4...v1.1.10.5) --- updated-dependencies: - dependency-name: org.xerial.snappy:snappy-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 4 ++-- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a18154b023a..50922b85a0c0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,7 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125)) - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752)) - Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126)) -- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.5 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206), [#10299](https://github.com/opensearch-project/OpenSearch/pull/10299)) - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208)) - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) - Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` @@ -143,4 +143,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 4aad96eddaeb5..7149efe27d694 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -72,5 +72,5 @@ dependencies { exclude group: "com.squareup.okio" } runtimeOnly "com.squareup.okio:okio:3.5.0" - runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.4" + runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.5" } From 28f185b347a3333c8670ca1a7bd7d0a85fed14e9 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 4 Oct 2023 17:12:05 -0500 Subject: [PATCH 103/388] Refactor multipart download to a more async model (#10349) * Refactor read context streams to async streams Signed-off-by: Kunal Kotwani * Refactor multipart download to a more async model The previous approach of kicking off the stream requests for all parts of a file did not work well for very large files. For example, a 20GiB file uploaded in 16MiB parts will consist of 1200+ parts. When we attempted to initiate streaming for all parts concurrently, some parts would hit a client timeout after 2 minutes without being able to get a connection due to the other parts not having been completed in that time frame. This refactoring adds yet another layer of indirection in order to allow the code that is actually writing the destination file to control the rate at which streams are started. This should allow for downloading files consisting of arbitrarily many parts at any connection speed. This commit also wires in the download rate limiter so that the `indices.recovery.max_bytes_per_sec` is properly honored. Signed-off-by: Andrew Ross --------- Signed-off-by: Kunal Kotwani Signed-off-by: Andrew Ross Co-authored-by: Kunal Kotwani --- .../repositories/s3/S3BlobContainer.java | 28 +--- .../s3/S3BlobStoreContainerTests.java | 4 +- .../opensearch/index/shard/IndexShardIT.java | 3 +- .../mocks/MockFsAsyncBlobContainer.java | 5 +- .../AsyncMultiStreamBlobContainer.java | 16 -- ...syncMultiStreamEncryptedBlobContainer.java | 6 +- .../blobstore/stream/read/ReadContext.java | 32 +++- .../read/listener/FileCompletionListener.java | 47 ------ .../stream/read/listener/FilePartWriter.java | 70 ++------- .../read/listener/ReadContextListener.java | 139 +++++++++++++++--- .../common/settings/ClusterSettings.java | 1 + .../org/opensearch/index/IndexService.java | 4 +- .../opensearch/index/shard/IndexShard.java | 39 +++-- .../opensearch/index/shard/StoreRecovery.java | 11 +- .../index/store/RemoteDirectory.java | 8 +- .../store/RemoteSegmentStoreDirectory.java | 18 ++- .../RemoteSegmentStoreDirectoryFactory.java | 15 +- .../indices/recovery/RecoverySettings.java | 25 ++++ .../RemoteStoreReplicationSource.java | 16 +- .../main/java/org/opensearch/node/Node.java | 6 +- .../blobstore/BlobStoreRepository.java | 6 +- .../org/opensearch/threadpool/ThreadPool.java | 6 + ...ultiStreamEncryptedBlobContainerTests.java | 10 +- .../listener/FileCompletionListenerTests.java | 58 -------- .../read/listener/FilePartWriterTests.java | 104 +------------ .../listener/ReadContextListenerTests.java | 54 +++++-- .../opensearch/index/IndexModuleTests.java | 3 +- .../RemoteStoreRefreshListenerTests.java | 4 +- ...moteSegmentStoreDirectoryFactoryTests.java | 7 +- .../RemoteSegmentStoreDirectoryTests.java | 29 ++-- .../snapshots/SnapshotResiliencyTests.java | 3 +- .../index/shard/IndexShardTestCase.java | 17 ++- .../recovery/DefaultRecoverySettings.java | 24 +++ 33 files changed, 400 insertions(+), 418 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListener.java delete mode 100644 server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListenerTests.java create mode 100644 test/framework/src/main/java/org/opensearch/indices/recovery/DefaultRecoverySettings.java diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index c6ae58371e15c..fcfccf50ad326 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -241,37 +241,23 @@ public void readBlobAsync(String blobName, ActionListener listener) return; } - final List> blobPartInputStreamFutures = new ArrayList<>(); + final List blobPartInputStreamFutures = new ArrayList<>(); final long blobSize = blobMetadata.objectSize(); final Integer numberOfParts = blobMetadata.objectParts() == null ? null : blobMetadata.objectParts().totalPartsCount(); final String blobChecksum = blobMetadata.checksum().checksumCRC32(); if (numberOfParts == null) { - blobPartInputStreamFutures.add(getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, null)); + blobPartInputStreamFutures.add(() -> getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, null)); } else { // S3 multipart files use 1 to n indexing for (int partNumber = 1; partNumber <= numberOfParts; partNumber++) { - blobPartInputStreamFutures.add(getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, partNumber)); + final int innerPartNumber = partNumber; + blobPartInputStreamFutures.add( + () -> getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, innerPartNumber) + ); } } - - CompletableFuture.allOf(blobPartInputStreamFutures.toArray(CompletableFuture[]::new)) - .whenComplete((unused, partThrowable) -> { - if (partThrowable == null) { - listener.onResponse( - new ReadContext( - blobSize, - blobPartInputStreamFutures.stream().map(CompletableFuture::join).collect(Collectors.toList()), - blobChecksum - ) - ); - } else { - Exception ex = partThrowable.getCause() instanceof Exception - ? (Exception) partThrowable.getCause() - : new Exception(partThrowable.getCause()); - listener.onFailure(ex); - } - }); + listener.onResponse(new ReadContext(blobSize, blobPartInputStreamFutures, blobChecksum)); }); } catch (Exception ex) { listener.onFailure(SdkException.create("Error occurred while fetching blob parts from the repository", ex)); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index 9817d7cd520ef..2e54705e9cd78 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -969,7 +969,7 @@ public void testReadBlobAsyncMultiPart() throws Exception { assertEquals(objectSize, readContext.getBlobSize()); for (int partNumber = 1; partNumber < objectPartCount; partNumber++) { - InputStreamContainer inputStreamContainer = readContext.getPartStreams().get(partNumber); + InputStreamContainer inputStreamContainer = readContext.getPartStreams().get(partNumber).get().join(); final int offset = partNumber * partSize; assertEquals(partSize, inputStreamContainer.getContentLength()); assertEquals(offset, inputStreamContainer.getOffset()); @@ -1024,7 +1024,7 @@ public void testReadBlobAsyncSinglePart() throws Exception { assertEquals(checksum, readContext.getBlobChecksum()); assertEquals(objectSize, readContext.getBlobSize()); - InputStreamContainer inputStreamContainer = readContext.getPartStreams().stream().findFirst().get(); + InputStreamContainer inputStreamContainer = readContext.getPartStreams().stream().findFirst().get().get().join(); assertEquals(objectSize, inputStreamContainer.getContentLength()); assertEquals(0, inputStreamContainer.getOffset()); assertEquals(objectSize, inputStreamContainer.getInputStream().readAllBytes().length); diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 07f85496f13cf..c394a1f631690 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -712,7 +712,8 @@ public static final IndexShard newIndexShard( null, null, () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, - nodeId + nodeId, + null ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java index 079753de95680..36987ac2d4991 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java @@ -27,6 +27,7 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -124,11 +125,11 @@ public void readBlobAsync(String blobName, ActionListener listener) long contentLength = listBlobs().get(blobName).length(); long partSize = contentLength / 10; int numberOfParts = (int) ((contentLength % partSize) == 0 ? contentLength / partSize : (contentLength / partSize) + 1); - List blobPartStreams = new ArrayList<>(); + List blobPartStreams = new ArrayList<>(); for (int partNumber = 0; partNumber < numberOfParts; partNumber++) { long offset = partNumber * partSize; InputStreamContainer blobPartStream = new InputStreamContainer(readBlob(blobName, offset, partSize), partSize, offset); - blobPartStreams.add(blobPartStream); + blobPartStreams.add(() -> CompletableFuture.completedFuture(blobPartStream)); } ReadContext blobReadContext = new ReadContext(contentLength, blobPartStreams, null); listener.onResponse(blobReadContext); diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java index e73a9f5cd0bc9..97f304d776f5c 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java @@ -10,13 +10,10 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.blobstore.stream.read.ReadContext; -import org.opensearch.common.blobstore.stream.read.listener.ReadContextListener; import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.core.action.ActionListener; -import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import java.nio.file.Path; /** * An extension of {@link BlobContainer} that adds {@link AsyncMultiStreamBlobContainer#asyncBlobUpload} to allow @@ -45,19 +42,6 @@ public interface AsyncMultiStreamBlobContainer extends BlobContainer { @ExperimentalApi void readBlobAsync(String blobName, ActionListener listener); - /** - * Asynchronously downloads the blob to the specified location using an executor from the thread pool. - * @param blobName The name of the blob for which needs to be downloaded. - * @param fileLocation The path on local disk where the blob needs to be downloaded. - * @param threadPool The threadpool instance which will provide the executor for performing a multipart download. - * @param completionListener Listener which will be notified when the download is complete. - */ - @ExperimentalApi - default void asyncBlobDownload(String blobName, Path fileLocation, ThreadPool threadPool, ActionListener completionListener) { - ReadContextListener readContextListener = new ReadContextListener(blobName, fileLocation, threadPool, completionListener); - readBlobAsync(blobName, readContextListener); - } - /* * Wether underlying blobContainer can verify integrity of data after transfer. If true and if expected * checksum is provided in WriteContext, then the checksum of transferred data is compared with expected checksum diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java index c64dc6b9e3ae4..82bc7a0baed50 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java @@ -144,8 +144,10 @@ public long getBlobSize() { } @Override - public List getPartStreams() { - return super.getPartStreams().stream().map(this::decryptInputStreamContainer).collect(Collectors.toList()); + public List getPartStreams() { + return super.getPartStreams().stream() + .map(supplier -> (StreamPartCreator) () -> supplier.get().thenApply(this::decryptInputStreamContainer)) + .collect(Collectors.toUnmodifiableList()); } /** diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java index 2c305fb03c475..4bdce11ff4f9a 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java +++ b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java @@ -12,6 +12,8 @@ import org.opensearch.common.io.InputStreamContainer; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; /** * ReadContext is used to encapsulate all data needed by BlobContainer#readBlobAsync @@ -19,18 +21,18 @@ @ExperimentalApi public class ReadContext { private final long blobSize; - private final List partStreams; + private final List asyncPartStreams; private final String blobChecksum; - public ReadContext(long blobSize, List partStreams, String blobChecksum) { + public ReadContext(long blobSize, List asyncPartStreams, String blobChecksum) { this.blobSize = blobSize; - this.partStreams = partStreams; + this.asyncPartStreams = asyncPartStreams; this.blobChecksum = blobChecksum; } public ReadContext(ReadContext readContext) { this.blobSize = readContext.blobSize; - this.partStreams = readContext.partStreams; + this.asyncPartStreams = readContext.asyncPartStreams; this.blobChecksum = readContext.blobChecksum; } @@ -39,14 +41,30 @@ public String getBlobChecksum() { } public int getNumberOfParts() { - return partStreams.size(); + return asyncPartStreams.size(); } public long getBlobSize() { return blobSize; } - public List getPartStreams() { - return partStreams; + public List getPartStreams() { + return asyncPartStreams; + } + + /** + * Functional interface defining an instance that can create an async action + * to create a part of an object represented as an InputStreamContainer. + */ + @FunctionalInterface + public interface StreamPartCreator extends Supplier> { + /** + * Kicks off a async process to start streaming. + * + * @return When the returned future is completed, streaming has + * just begun. Clients must fully consume the resulting stream. + */ + @Override + CompletableFuture get(); } } diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListener.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListener.java deleted file mode 100644 index aadd6e2ab304e..0000000000000 --- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListener.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.common.blobstore.stream.read.listener; - -import org.opensearch.common.annotation.InternalApi; -import org.opensearch.core.action.ActionListener; - -import java.util.concurrent.atomic.AtomicInteger; - -/** - * FileCompletionListener listens for completion of fetch on all the streams for a file, where - * individual streams are handled using {@link FilePartWriter}. The {@link FilePartWriter}(s) - * hold a reference to the file completion listener to be notified. - */ -@InternalApi -class FileCompletionListener implements ActionListener { - - private final int numberOfParts; - private final String fileName; - private final AtomicInteger completedPartsCount; - private final ActionListener completionListener; - - public FileCompletionListener(int numberOfParts, String fileName, ActionListener completionListener) { - this.completedPartsCount = new AtomicInteger(); - this.numberOfParts = numberOfParts; - this.fileName = fileName; - this.completionListener = completionListener; - } - - @Override - public void onResponse(Integer unused) { - if (completedPartsCount.incrementAndGet() == numberOfParts) { - completionListener.onResponse(fileName); - } - } - - @Override - public void onFailure(Exception e) { - completionListener.onFailure(e); - } -} diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriter.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriter.java index 84fd7ed9ffebf..1a403200249cd 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriter.java +++ b/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriter.java @@ -8,83 +8,37 @@ package org.opensearch.common.blobstore.stream.read.listener; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.io.Channels; import org.opensearch.common.io.InputStreamContainer; -import org.opensearch.core.action.ActionListener; import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.UnaryOperator; /** * FilePartWriter transfers the provided stream into the specified file path using a {@link FileChannel} - * instance. It performs offset based writes to the file and notifies the {@link FileCompletionListener} on completion. + * instance. */ @InternalApi -class FilePartWriter implements Runnable { - - private final int partNumber; - private final InputStreamContainer blobPartStreamContainer; - private final Path fileLocation; - private final AtomicBoolean anyPartStreamFailed; - private final ActionListener fileCompletionListener; - private static final Logger logger = LogManager.getLogger(FilePartWriter.class); - +class FilePartWriter { // 8 MB buffer for transfer private static final int BUFFER_SIZE = 8 * 1024 * 2024; - public FilePartWriter( - int partNumber, - InputStreamContainer blobPartStreamContainer, - Path fileLocation, - AtomicBoolean anyPartStreamFailed, - ActionListener fileCompletionListener - ) { - this.partNumber = partNumber; - this.blobPartStreamContainer = blobPartStreamContainer; - this.fileLocation = fileLocation; - this.anyPartStreamFailed = anyPartStreamFailed; - this.fileCompletionListener = fileCompletionListener; - } - - @Override - public void run() { - // Ensures no writes to the file if any stream fails. - if (anyPartStreamFailed.get() == false) { - try (FileChannel outputFileChannel = FileChannel.open(fileLocation, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { - try (InputStream inputStream = blobPartStreamContainer.getInputStream()) { - long streamOffset = blobPartStreamContainer.getOffset(); - final byte[] buffer = new byte[BUFFER_SIZE]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - Channels.writeToChannel(buffer, 0, bytesRead, outputFileChannel, streamOffset); - streamOffset += bytesRead; - } + public static void write(Path fileLocation, InputStreamContainer stream, UnaryOperator rateLimiter) throws IOException { + try (FileChannel outputFileChannel = FileChannel.open(fileLocation, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { + try (InputStream inputStream = rateLimiter.apply(stream.getInputStream())) { + long streamOffset = stream.getOffset(); + final byte[] buffer = new byte[BUFFER_SIZE]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + Channels.writeToChannel(buffer, 0, bytesRead, outputFileChannel, streamOffset); + streamOffset += bytesRead; } - } catch (IOException e) { - processFailure(e); - return; } - fileCompletionListener.onResponse(partNumber); - } - } - - void processFailure(Exception e) { - try { - Files.deleteIfExists(fileLocation); - } catch (IOException ex) { - // Die silently - logger.info("Failed to delete file {} on stream failure: {}", fileLocation, ex); - } - if (anyPartStreamFailed.getAndSet(true) == false) { - fileCompletionListener.onFailure(e); } } } diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java index 4338bddb3fbe7..2914fd0c440fa 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java +++ b/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java @@ -10,51 +10,73 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.GroupedActionListener; import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.core.action.ActionListener; import org.opensearch.threadpool.ThreadPool; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.UnaryOperator; /** * ReadContextListener orchestrates the async file fetch from the {@link org.opensearch.common.blobstore.BlobContainer} - * using a {@link ReadContext} callback. On response, it spawns off the download using multiple streams which are - * spread across a {@link ThreadPool} executor. + * using a {@link ReadContext} callback. On response, it spawns off the download using multiple streams. */ @InternalApi public class ReadContextListener implements ActionListener { + private static final Logger logger = LogManager.getLogger(ReadContextListener.class); - private final String fileName; + private final String blobName; private final Path fileLocation; - private final ThreadPool threadPool; private final ActionListener completionListener; - private static final Logger logger = LogManager.getLogger(ReadContextListener.class); + private final ThreadPool threadPool; + private final UnaryOperator rateLimiter; + private final int maxConcurrentStreams; - public ReadContextListener(String fileName, Path fileLocation, ThreadPool threadPool, ActionListener completionListener) { - this.fileName = fileName; + public ReadContextListener( + String blobName, + Path fileLocation, + ActionListener completionListener, + ThreadPool threadPool, + UnaryOperator rateLimiter, + int maxConcurrentStreams + ) { + this.blobName = blobName; this.fileLocation = fileLocation; - this.threadPool = threadPool; this.completionListener = completionListener; + this.threadPool = threadPool; + this.rateLimiter = rateLimiter; + this.maxConcurrentStreams = maxConcurrentStreams; } @Override public void onResponse(ReadContext readContext) { - logger.trace("Streams received for blob {}", fileName); + logger.debug("Received {} parts for blob {}", readContext.getNumberOfParts(), blobName); final int numParts = readContext.getNumberOfParts(); - final AtomicBoolean anyPartStreamFailed = new AtomicBoolean(); - FileCompletionListener fileCompletionListener = new FileCompletionListener(numParts, fileName, completionListener); - - for (int partNumber = 0; partNumber < numParts; partNumber++) { - FilePartWriter filePartWriter = new FilePartWriter( - partNumber, - readContext.getPartStreams().get(partNumber), - fileLocation, - anyPartStreamFailed, - fileCompletionListener - ); - threadPool.executor(ThreadPool.Names.GENERIC).submit(filePartWriter); + final AtomicBoolean anyPartStreamFailed = new AtomicBoolean(false); + final GroupedActionListener groupedListener = new GroupedActionListener<>( + ActionListener.wrap(r -> completionListener.onResponse(blobName), completionListener::onFailure), + numParts + ); + final Queue queue = new ConcurrentLinkedQueue<>(readContext.getPartStreams()); + final StreamPartProcessor processor = new StreamPartProcessor( + queue, + anyPartStreamFailed, + fileLocation, + groupedListener, + threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY), + rateLimiter + ); + for (int i = 0; i < Math.min(maxConcurrentStreams, queue.size()); i++) { + processor.process(queue.poll()); } } @@ -62,4 +84,79 @@ public void onResponse(ReadContext readContext) { public void onFailure(Exception e) { completionListener.onFailure(e); } + + private static class StreamPartProcessor { + private static final RuntimeException CANCELED_PART_EXCEPTION = new RuntimeException( + "Canceled part download due to previous failure" + ); + private final Queue queue; + private final AtomicBoolean anyPartStreamFailed; + private final Path fileLocation; + private final GroupedActionListener completionListener; + private final Executor executor; + private final UnaryOperator rateLimiter; + + private StreamPartProcessor( + Queue queue, + AtomicBoolean anyPartStreamFailed, + Path fileLocation, + GroupedActionListener completionListener, + Executor executor, + UnaryOperator rateLimiter + ) { + this.queue = queue; + this.anyPartStreamFailed = anyPartStreamFailed; + this.fileLocation = fileLocation; + this.completionListener = completionListener; + this.executor = executor; + this.rateLimiter = rateLimiter; + } + + private void process(ReadContext.StreamPartCreator supplier) { + if (supplier == null) { + return; + } + supplier.get().whenCompleteAsync((blobPartStreamContainer, throwable) -> { + if (throwable != null) { + processFailure(throwable instanceof Exception ? (Exception) throwable : new RuntimeException(throwable)); + } else if (anyPartStreamFailed.get()) { + processFailure(CANCELED_PART_EXCEPTION); + } else { + try { + FilePartWriter.write(fileLocation, blobPartStreamContainer, rateLimiter); + completionListener.onResponse(fileLocation.toString()); + + // Upon successfully completing a file part, pull another + // file part off the queue to trigger asynchronous processing + process(queue.poll()); + } catch (Exception e) { + processFailure(e); + } + } + }, executor); + } + + private void processFailure(Exception e) { + if (anyPartStreamFailed.getAndSet(true) == false) { + completionListener.onFailure(e); + + // Drain the queue of pending part downloads. These can be discarded + // since they haven't started any work yet, but the listener must be + // notified for each part. + Object item = queue.poll(); + while (item != null) { + completionListener.onFailure(CANCELED_PART_EXCEPTION); + item = queue.poll(); + } + } else { + completionListener.onFailure(e); + } + try { + Files.deleteIfExists(fileLocation); + } catch (IOException ex) { + // Die silently + logger.info("Failed to delete file {} on stream failure: {}", fileLocation, ex); + } + } + } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 5261d40387dc6..4cd3490cffb4c 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -287,6 +287,7 @@ public void apply(Settings value, Settings current, Settings previous) { RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING, RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING, RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING, + RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING, ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING, ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING, ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index fdda8d4ce2497..df8e8070b8e03 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -89,6 +89,7 @@ import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogFactory; @@ -520,7 +521,8 @@ public synchronized IndexShard createShard( remoteStore, remoteStoreStatsTrackerFactory, clusterRemoteTranslogBufferIntervalSupplier, - nodeEnv.nodeId() + nodeEnv.nodeId(), + (RemoteSegmentStoreDirectoryFactory) remoteDirectoryFactory ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 5818b2d866854..4f08411c19b55 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -62,7 +62,6 @@ import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.opensearch.action.admin.indices.upgrade.post.UpgradeRequest; -import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.replication.PendingReplicationActions; import org.opensearch.action.support.replication.ReplicationResponse; @@ -162,6 +161,7 @@ import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.store.Store.MetadataSnapshot; import org.opensearch.index.store.StoreFileMetadata; @@ -341,6 +341,7 @@ Runnable getGlobalCheckpointSyncer() { private final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory; private final List internalRefreshListener = new ArrayList<>(); + private final RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory; public IndexShard( final ShardRouting shardRouting, @@ -368,7 +369,11 @@ public IndexShard( @Nullable final Store remoteStore, final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, final Supplier clusterRemoteTranslogBufferIntervalSupplier, - final String nodeId + final String nodeId, + // Wiring a directory factory here breaks some intended abstractions, but this remote directory + // factory is used not as a Lucene directory but instead to copy files from a remote store when + // restoring a shallow snapshot. + @Nullable final RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory ) throws IOException { super(shardRouting.shardId(), indexSettings); assert shardRouting.initializing(); @@ -464,6 +469,7 @@ public boolean shouldCache(Query query) { ? false : mapperService.documentMapper().mappers().containsTimeStampField(); this.remoteStoreStatsTrackerFactory = remoteStoreStatsTrackerFactory; + this.remoteSegmentStoreDirectoryFactory = remoteSegmentStoreDirectoryFactory; } public ThreadPool getThreadPool() { @@ -2696,7 +2702,7 @@ public void restoreFromRemoteStore(ActionListener listener) { public void restoreFromSnapshotAndRemoteStore( Repository repository, - RepositoriesService repositoriesService, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, ActionListener listener ) { try { @@ -2704,7 +2710,7 @@ public void restoreFromSnapshotAndRemoteStore( assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + recoveryState.getRecoverySource(); StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); - storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, repositoriesService, listener, threadPool); + storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, remoteSegmentStoreDirectoryFactory, listener); } catch (Exception e) { listener.onFailure(e); } @@ -3544,7 +3550,7 @@ public void startRecovery( "from snapshot and remote store", recoveryState, recoveryListener, - l -> restoreFromSnapshotAndRemoteStore(repositoriesService.repository(repo), repositoriesService, l) + l -> restoreFromSnapshotAndRemoteStore(repositoriesService.repository(repo), remoteSegmentStoreDirectoryFactory, l) ); // indicesService.indexService(shardRouting.shardId().getIndex()).addMetadataListener(); } else { @@ -4921,24 +4927,17 @@ private void downloadSegments( RemoteSegmentStoreDirectory targetRemoteDirectory, Set toDownloadSegments, final Runnable onFileSync - ) { - final PlainActionFuture completionListener = PlainActionFuture.newFuture(); - final GroupedActionListener batchDownloadListener = new GroupedActionListener<>( - ActionListener.map(completionListener, v -> null), - toDownloadSegments.size() - ); - - final ActionListener segmentsDownloadListener = ActionListener.map(batchDownloadListener, fileName -> { + ) throws IOException { + final Path indexPath = store.shardPath() == null ? null : store.shardPath().resolveIndex(); + for (String segment : toDownloadSegments) { + final PlainActionFuture segmentListener = PlainActionFuture.newFuture(); + sourceRemoteDirectory.copyTo(segment, storeDirectory, indexPath, segmentListener); + segmentListener.actionGet(); onFileSync.run(); if (targetRemoteDirectory != null) { - targetRemoteDirectory.copyFrom(storeDirectory, fileName, fileName, IOContext.DEFAULT); + targetRemoteDirectory.copyFrom(storeDirectory, segment, segment, IOContext.DEFAULT); } - return null; - }); - - final Path indexPath = store.shardPath() == null ? null : store.shardPath().resolveIndex(); - toDownloadSegments.forEach(file -> { sourceRemoteDirectory.copyTo(file, storeDirectory, indexPath, segmentsDownloadListener); }); - completionListener.actionGet(); + } } private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) { diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index c0211e1257c8e..762aab51469d0 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -70,9 +70,7 @@ import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.repositories.IndexId; -import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; -import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.nio.channels.FileChannel; @@ -362,9 +360,8 @@ void recoverFromRepository(final IndexShard indexShard, Repository repository, A void recoverFromSnapshotAndRemoteStore( final IndexShard indexShard, Repository repository, - RepositoriesService repositoriesService, - ActionListener listener, - ThreadPool threadPool + RemoteSegmentStoreDirectoryFactory directoryFactory, + ActionListener listener ) { try { if (canRecover(indexShard)) { @@ -392,10 +389,6 @@ void recoverFromSnapshotAndRemoteStore( remoteStoreRepository = shallowCopyShardMetadata.getRemoteStoreRepository(); } - RemoteSegmentStoreDirectoryFactory directoryFactory = new RemoteSegmentStoreDirectoryFactory( - () -> repositoriesService, - threadPool - ); RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( remoteStoreRepository, indexUUID, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index 594b7f99cd85a..eb75c39532d71 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -62,9 +62,9 @@ public class RemoteDirectory extends Directory { protected final BlobContainer blobContainer; private static final Logger logger = LogManager.getLogger(RemoteDirectory.class); - protected final UnaryOperator uploadRateLimiter; + private final UnaryOperator uploadRateLimiter; - protected final UnaryOperator downloadRateLimiter; + private final UnaryOperator downloadRateLimiter; /** * Number of bytes in the segment file to store checksum @@ -333,6 +333,10 @@ public boolean copyFrom( return false; } + protected UnaryOperator getDownloadRateLimiter() { + return downloadRateLimiter; + } + private void uploadBlob( Directory from, String src, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 21a84f2b8c903..a97b22360716c 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -25,6 +25,7 @@ import org.apache.lucene.util.Version; import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.stream.read.listener.ReadContextListener; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.logging.Loggers; @@ -37,6 +38,7 @@ import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.threadpool.ThreadPool; @@ -90,6 +92,8 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement private final ThreadPool threadPool; + private final RecoverySettings recoverySettings; + /** * Keeps track of local segment filename to uploaded filename along with other attributes like checksum. * This map acts as a cache layer for uploaded segment filenames which helps avoid calling listAll() each time. @@ -122,13 +126,15 @@ public RemoteSegmentStoreDirectory( RemoteDirectory remoteMetadataDirectory, RemoteStoreLockManager mdLockManager, ThreadPool threadPool, - ShardId shardId + ShardId shardId, + RecoverySettings recoverySettings ) throws IOException { super(remoteDataDirectory); this.remoteDataDirectory = remoteDataDirectory; this.remoteMetadataDirectory = remoteMetadataDirectory; this.mdLockManager = mdLockManager; this.threadPool = threadPool; + this.recoverySettings = recoverySettings; this.logger = Loggers.getLogger(getClass(), shardId); init(); } @@ -488,7 +494,15 @@ public void copyTo(String source, Directory destinationDirectory, Path destinati if (destinationPath != null && remoteDataDirectory.getBlobContainer() instanceof AsyncMultiStreamBlobContainer) { final AsyncMultiStreamBlobContainer blobContainer = (AsyncMultiStreamBlobContainer) remoteDataDirectory.getBlobContainer(); final Path destinationFilePath = destinationPath.resolve(source); - blobContainer.asyncBlobDownload(blobName, destinationFilePath, threadPool, fileCompletionListener); + final ReadContextListener readContextListener = new ReadContextListener( + blobName, + destinationFilePath, + fileCompletionListener, + threadPool, + remoteDataDirectory.getDownloadRateLimiter(), + recoverySettings.getMaxConcurrentRemoteStoreStreams() + ); + blobContainer.readBlobAsync(blobName, readContextListener); } else { // Fallback to older mechanism of downloading the file try { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 490b07e441702..cc55380894ecd 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -15,6 +15,7 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -34,12 +35,18 @@ public class RemoteSegmentStoreDirectoryFactory implements IndexStorePlugin.Dire private static final String SEGMENTS = "segments"; private final Supplier repositoriesService; + private final RecoverySettings recoverySettings; private final ThreadPool threadPool; - public RemoteSegmentStoreDirectoryFactory(Supplier repositoriesService, ThreadPool threadPool) { + public RemoteSegmentStoreDirectoryFactory( + Supplier repositoriesService, + ThreadPool threadPool, + RecoverySettings recoverySettings + ) { this.repositoriesService = repositoriesService; this.threadPool = threadPool; + this.recoverySettings = recoverySettings; } @Override @@ -71,13 +78,9 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s String.valueOf(shardId.id()) ); - return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool, shardId); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool, shardId, recoverySettings); } catch (RepositoryMissingException e) { throw new IllegalArgumentException("Repository should be created before creating index with remote_store enabled setting", e); } } - - private RemoteDirectory createRemoteDirectory(BlobStoreRepository repository, BlobPath commonBlobPath, String extension) { - return new RemoteDirectory(repository.blobStore().blobContainer(commonBlobPath.add(extension))); - } } diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java index e2346ae078339..ed9755bf824ea 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java @@ -84,6 +84,17 @@ public class RecoverySettings { Property.NodeScope ); + /** + * Controls the maximum number of streams that can be started concurrently when downloading from the remote store. + */ + public static final Setting INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING = Setting.intSetting( + "indices.recovery.max_concurrent_remote_store_streams", + 20, + 1, + Property.Dynamic, + Property.NodeScope + ); + /** * how long to wait before retrying after issues cause by cluster state syncing between nodes * i.e., local node is not yet known on remote node, remote shard not yet started etc. @@ -149,6 +160,7 @@ public class RecoverySettings { private volatile ByteSizeValue maxBytesPerSec; private volatile int maxConcurrentFileChunks; private volatile int maxConcurrentOperations; + private volatile int maxConcurrentRemoteStoreStreams; private volatile SimpleRateLimiter rateLimiter; private volatile TimeValue retryDelayStateSync; private volatile TimeValue retryDelayNetwork; @@ -163,6 +175,7 @@ public RecoverySettings(Settings settings, ClusterSettings clusterSettings) { this.retryDelayStateSync = INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.get(settings); this.maxConcurrentFileChunks = INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING.get(settings); this.maxConcurrentOperations = INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING.get(settings); + this.maxConcurrentRemoteStoreStreams = INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING.get(settings); // doesn't have to be fast as nodes are reconnected every 10s by default (see InternalClusterService.ReconnectToNodes) // and we want to give the cluster-manager time to remove a faulty node this.retryDelayNetwork = INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.get(settings); @@ -184,6 +197,10 @@ public RecoverySettings(Settings settings, ClusterSettings clusterSettings) { clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING, this::setMaxBytesPerSec); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING, this::setMaxConcurrentFileChunks); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING, this::setMaxConcurrentOperations); + clusterSettings.addSettingsUpdateConsumer( + INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING, + this::setMaxConcurrentRemoteStoreStreams + ); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING, this::setRetryDelayStateSync); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING, this::setRetryDelayNetwork); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING, this::setInternalActionTimeout); @@ -279,4 +296,12 @@ public int getMaxConcurrentOperations() { private void setMaxConcurrentOperations(int maxConcurrentOperations) { this.maxConcurrentOperations = maxConcurrentOperations; } + + public int getMaxConcurrentRemoteStoreStreams() { + return this.maxConcurrentRemoteStoreStreams; + } + + private void setMaxConcurrentRemoteStoreStreams(int maxConcurrentRemoteStoreStreams) { + this.maxConcurrentRemoteStoreStreams = maxConcurrentRemoteStoreStreams; + } } diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java index aeb690465905f..e17c5293c38ac 100644 --- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java @@ -14,7 +14,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.util.Version; -import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.core.action.ActionListener; import org.opensearch.index.shard.IndexShard; @@ -141,14 +141,12 @@ private void downloadSegments( ActionListener completionListener ) { final Path indexPath = shardPath == null ? null : shardPath.resolveIndex(); - final GroupedActionListener batchDownloadListener = new GroupedActionListener<>( - ActionListener.map(completionListener, v -> new GetSegmentFilesResponse(toDownloadSegments)), - toDownloadSegments.size() - ); - ActionListener segmentsDownloadListener = ActionListener.map(batchDownloadListener, result -> null); - toDownloadSegments.forEach( - fileMetadata -> remoteStoreDirectory.copyTo(fileMetadata.name(), storeDirectory, indexPath, segmentsDownloadListener) - ); + for (StoreFileMetadata storeFileMetadata : toDownloadSegments) { + final PlainActionFuture segmentListener = PlainActionFuture.newFuture(); + remoteStoreDirectory.copyTo(storeFileMetadata.name(), storeDirectory, indexPath, segmentListener); + segmentListener.actionGet(); + } + completionListener.onResponse(new GetSegmentFilesResponse(toDownloadSegments)); } @Override diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 729b38ca27394..02f6bdd5ad24c 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -757,9 +757,12 @@ protected Node( rerouteServiceReference.set(rerouteService); clusterService.setRerouteService(rerouteService); + final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); + final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( repositoriesServiceReference::get, - threadPool + threadPool, + recoverySettings ); final SearchRequestStats searchRequestStats = new SearchRequestStats(); @@ -951,7 +954,6 @@ protected Node( transportService.getTaskManager() ); - final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); RepositoriesModule repositoriesModule = new RepositoriesModule( this.environment, pluginsService.filterPlugins(RepositoryPlugin.class), diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 3481e43cf4c72..41ad357eaeed9 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -1126,7 +1126,8 @@ private void executeStaleShardDelete( // see https://github.com/opensearch-project/OpenSearch/issues/8469 new RemoteSegmentStoreDirectoryFactory( remoteStoreLockManagerFactory.getRepositoriesService(), - threadPool + threadPool, + recoverySettings ).newDirectory( remoteStoreRepoForIndex, indexUUID, @@ -1596,7 +1597,8 @@ private void executeOneStaleIndexDelete( // see https://github.com/opensearch-project/OpenSearch/issues/8469 new RemoteSegmentStoreDirectoryFactory( remoteStoreLockManagerFactory.getRepositoriesService(), - threadPool + threadPool, + recoverySettings ).newDirectory( remoteStoreRepoForIndex, indexUUID, diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 8375ac34972af..ecb5b2cef58ac 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -115,6 +115,7 @@ public static class Names { public static final String TRANSLOG_SYNC = "translog_sync"; public static final String REMOTE_PURGE = "remote_purge"; public static final String REMOTE_REFRESH_RETRY = "remote_refresh_retry"; + public static final String REMOTE_RECOVERY = "remote_recovery"; public static final String INDEX_SEARCHER = "index_searcher"; } @@ -184,6 +185,7 @@ public static ThreadPoolType fromType(String type) { map.put(Names.TRANSLOG_SYNC, ThreadPoolType.FIXED); map.put(Names.REMOTE_PURGE, ThreadPoolType.SCALING); map.put(Names.REMOTE_REFRESH_RETRY, ThreadPoolType.SCALING); + map.put(Names.REMOTE_RECOVERY, ThreadPoolType.SCALING); if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { map.put(Names.INDEX_SEARCHER, ThreadPoolType.RESIZABLE); } @@ -269,6 +271,10 @@ public ThreadPool( Names.REMOTE_REFRESH_RETRY, new ScalingExecutorBuilder(Names.REMOTE_REFRESH_RETRY, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5)) ); + builders.put( + Names.REMOTE_RECOVERY, + new ScalingExecutorBuilder(Names.REMOTE_RECOVERY, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5)) + ); if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { builders.put( Names.INDEX_SEARCHER, diff --git a/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java index 947a4f9b1c9ab..1780819390052 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java @@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.function.UnaryOperator; import org.mockito.Mockito; @@ -51,10 +52,12 @@ public void testReadBlobAsync() throws Exception { // Objects needed for API call final byte[] data = new byte[size]; Randomness.get().nextBytes(data); + final InputStreamContainer inputStreamContainer = new InputStreamContainer(new ByteArrayInputStream(data), data.length, 0); final ListenerTestUtils.CountingCompletionListener completionListener = new ListenerTestUtils.CountingCompletionListener<>(); - final ReadContext readContext = new ReadContext(size, List.of(inputStreamContainer), null); + final CompletableFuture streamContainerFuture = CompletableFuture.completedFuture(inputStreamContainer); + final ReadContext readContext = new ReadContext(size, List.of(() -> streamContainerFuture), null); Mockito.doAnswer(invocation -> { ActionListener readContextActionListener = invocation.getArgument(1); @@ -76,7 +79,7 @@ public void testReadBlobAsync() throws Exception { assertEquals(1, response.getNumberOfParts()); assertEquals(size, response.getBlobSize()); - InputStreamContainer responseContainer = response.getPartStreams().get(0); + InputStreamContainer responseContainer = response.getPartStreams().get(0).get().join(); assertEquals(0, responseContainer.getOffset()); assertEquals(size, responseContainer.getContentLength()); assertEquals(100, responseContainer.getInputStream().available()); @@ -99,7 +102,8 @@ public void testReadBlobAsyncException() throws Exception { final InputStreamContainer inputStreamContainer = new InputStreamContainer(new ByteArrayInputStream(data), data.length, 0); final ListenerTestUtils.CountingCompletionListener completionListener = new ListenerTestUtils.CountingCompletionListener<>(); - final ReadContext readContext = new ReadContext(size, List.of(inputStreamContainer), null); + final CompletableFuture streamContainerFuture = CompletableFuture.completedFuture(inputStreamContainer); + final ReadContext readContext = new ReadContext(size, List.of(() -> streamContainerFuture), null); Mockito.doAnswer(invocation -> { ActionListener readContextActionListener = invocation.getArgument(1); diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListenerTests.java deleted file mode 100644 index fa13d90f42fa6..0000000000000 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FileCompletionListenerTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.common.blobstore.stream.read.listener; - -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; - -import static org.opensearch.common.blobstore.stream.read.listener.ListenerTestUtils.CountingCompletionListener; - -public class FileCompletionListenerTests extends OpenSearchTestCase { - - public void testFileCompletionListener() { - int numStreams = 10; - String fileName = "test_segment_file"; - CountingCompletionListener completionListener = new CountingCompletionListener(); - FileCompletionListener fileCompletionListener = new FileCompletionListener(numStreams, fileName, completionListener); - - for (int stream = 0; stream < numStreams; stream++) { - // Ensure completion listener called only when all streams are completed - assertEquals(0, completionListener.getResponseCount()); - fileCompletionListener.onResponse(null); - } - - assertEquals(1, completionListener.getResponseCount()); - assertEquals(fileName, completionListener.getResponse()); - } - - public void testFileCompletionListenerFailure() { - int numStreams = 10; - String fileName = "test_segment_file"; - CountingCompletionListener completionListener = new CountingCompletionListener(); - FileCompletionListener fileCompletionListener = new FileCompletionListener(numStreams, fileName, completionListener); - - // Fail the listener initially - IOException exception = new IOException(); - fileCompletionListener.onFailure(exception); - - for (int stream = 0; stream < numStreams - 1; stream++) { - assertEquals(0, completionListener.getResponseCount()); - fileCompletionListener.onResponse(null); - } - - assertEquals(1, completionListener.getFailureCount()); - assertEquals(exception, completionListener.getException()); - assertEquals(0, completionListener.getResponseCount()); - - fileCompletionListener.onFailure(exception); - assertEquals(2, completionListener.getFailureCount()); - assertEquals(exception, completionListener.getException()); - } -} diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriterTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriterTests.java index 811566eb5767b..f2a758b9bbe10 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriterTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/FilePartWriterTests.java @@ -13,14 +13,11 @@ import org.junit.Before; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.opensearch.common.blobstore.stream.read.listener.ListenerTestUtils.CountingCompletionListener; +import java.util.function.UnaryOperator; public class FilePartWriterTests extends OpenSearchTestCase { @@ -34,130 +31,37 @@ public void init() throws Exception { public void testFilePartWriter() throws Exception { Path segmentFilePath = path.resolve(UUID.randomUUID().toString()); int contentLength = 100; - int partNumber = 1; InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(contentLength)); InputStreamContainer inputStreamContainer = new InputStreamContainer(inputStream, inputStream.available(), 0); - AtomicBoolean anyStreamFailed = new AtomicBoolean(); - CountingCompletionListener fileCompletionListener = new CountingCompletionListener<>(); - FilePartWriter filePartWriter = new FilePartWriter( - partNumber, - inputStreamContainer, - segmentFilePath, - anyStreamFailed, - fileCompletionListener - ); - filePartWriter.run(); + FilePartWriter.write(segmentFilePath, inputStreamContainer, UnaryOperator.identity()); assertTrue(Files.exists(segmentFilePath)); assertEquals(contentLength, Files.size(segmentFilePath)); - assertEquals(1, fileCompletionListener.getResponseCount()); - assertEquals(Integer.valueOf(partNumber), fileCompletionListener.getResponse()); } public void testFilePartWriterWithOffset() throws Exception { Path segmentFilePath = path.resolve(UUID.randomUUID().toString()); int contentLength = 100; int offset = 10; - int partNumber = 1; InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(contentLength)); InputStreamContainer inputStreamContainer = new InputStreamContainer(inputStream, inputStream.available(), offset); - AtomicBoolean anyStreamFailed = new AtomicBoolean(); - CountingCompletionListener fileCompletionListener = new CountingCompletionListener<>(); - FilePartWriter filePartWriter = new FilePartWriter( - partNumber, - inputStreamContainer, - segmentFilePath, - anyStreamFailed, - fileCompletionListener - ); - filePartWriter.run(); + FilePartWriter.write(segmentFilePath, inputStreamContainer, UnaryOperator.identity()); assertTrue(Files.exists(segmentFilePath)); assertEquals(contentLength + offset, Files.size(segmentFilePath)); - assertEquals(1, fileCompletionListener.getResponseCount()); - assertEquals(Integer.valueOf(partNumber), fileCompletionListener.getResponse()); } public void testFilePartWriterLargeInput() throws Exception { Path segmentFilePath = path.resolve(UUID.randomUUID().toString()); int contentLength = 20 * 1024 * 1024; - int partNumber = 1; InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(contentLength)); InputStreamContainer inputStreamContainer = new InputStreamContainer(inputStream, contentLength, 0); - AtomicBoolean anyStreamFailed = new AtomicBoolean(); - CountingCompletionListener fileCompletionListener = new CountingCompletionListener<>(); - FilePartWriter filePartWriter = new FilePartWriter( - partNumber, - inputStreamContainer, - segmentFilePath, - anyStreamFailed, - fileCompletionListener - ); - filePartWriter.run(); + FilePartWriter.write(segmentFilePath, inputStreamContainer, UnaryOperator.identity()); assertTrue(Files.exists(segmentFilePath)); assertEquals(contentLength, Files.size(segmentFilePath)); - - assertEquals(1, fileCompletionListener.getResponseCount()); - assertEquals(Integer.valueOf(partNumber), fileCompletionListener.getResponse()); - } - - public void testFilePartWriterException() throws Exception { - Path segmentFilePath = path.resolve(UUID.randomUUID().toString()); - int contentLength = 100; - int partNumber = 1; - InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(contentLength)); - InputStreamContainer inputStreamContainer = new InputStreamContainer(inputStream, contentLength, 0); - AtomicBoolean anyStreamFailed = new AtomicBoolean(); - CountingCompletionListener fileCompletionListener = new CountingCompletionListener<>(); - - IOException ioException = new IOException(); - FilePartWriter filePartWriter = new FilePartWriter( - partNumber, - inputStreamContainer, - segmentFilePath, - anyStreamFailed, - fileCompletionListener - ); - assertFalse(anyStreamFailed.get()); - filePartWriter.processFailure(ioException); - - assertTrue(anyStreamFailed.get()); - assertFalse(Files.exists(segmentFilePath)); - - // Fail stream again to simulate another stream failure for same file - filePartWriter.processFailure(ioException); - - assertTrue(anyStreamFailed.get()); - assertFalse(Files.exists(segmentFilePath)); - - assertEquals(0, fileCompletionListener.getResponseCount()); - assertEquals(1, fileCompletionListener.getFailureCount()); - assertEquals(ioException, fileCompletionListener.getException()); - } - - public void testFilePartWriterStreamFailed() throws Exception { - Path segmentFilePath = path.resolve(UUID.randomUUID().toString()); - int contentLength = 100; - int partNumber = 1; - InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(contentLength)); - InputStreamContainer inputStreamContainer = new InputStreamContainer(inputStream, inputStream.available(), 0); - AtomicBoolean anyStreamFailed = new AtomicBoolean(true); - CountingCompletionListener fileCompletionListener = new CountingCompletionListener<>(); - - FilePartWriter filePartWriter = new FilePartWriter( - partNumber, - inputStreamContainer, - segmentFilePath, - anyStreamFailed, - fileCompletionListener - ); - filePartWriter.run(); - - assertFalse(Files.exists(segmentFilePath)); - assertEquals(0, fileCompletionListener.getResponseCount()); } } diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java index 21b7b47390a9b..7e4c96cbadcda 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java @@ -29,7 +29,9 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.function.UnaryOperator; import static org.opensearch.common.blobstore.stream.read.listener.ListenerTestUtils.CountingCompletionListener; @@ -46,6 +48,7 @@ public class ReadContextListenerTests extends OpenSearchTestCase { private static final int NUMBER_OF_PARTS = 5; private static final int PART_SIZE = 10; private static final String TEST_SEGMENT_FILE = "test_segment_file"; + private static final int MAX_CONCURRENT_STREAMS = 10; @BeforeClass public static void setup() { @@ -64,10 +67,17 @@ public void init() throws Exception { public void testReadContextListener() throws InterruptedException, IOException { Path fileLocation = path.resolve(UUID.randomUUID().toString()); - List blobPartStreams = initializeBlobPartStreams(); + List blobPartStreams = initializeBlobPartStreams(); CountDownLatch countDownLatch = new CountDownLatch(1); ActionListener completionListener = new LatchedActionListener<>(new PlainActionFuture<>(), countDownLatch); - ReadContextListener readContextListener = new ReadContextListener(TEST_SEGMENT_FILE, fileLocation, threadPool, completionListener); + ReadContextListener readContextListener = new ReadContextListener( + TEST_SEGMENT_FILE, + fileLocation, + completionListener, + threadPool, + UnaryOperator.identity(), + MAX_CONCURRENT_STREAMS + ); ReadContext readContext = new ReadContext((long) PART_SIZE * NUMBER_OF_PARTS, blobPartStreams, null); readContextListener.onResponse(readContext); @@ -79,10 +89,17 @@ public void testReadContextListener() throws InterruptedException, IOException { public void testReadContextListenerFailure() throws Exception { Path fileLocation = path.resolve(UUID.randomUUID().toString()); - List blobPartStreams = initializeBlobPartStreams(); + List blobPartStreams = initializeBlobPartStreams(); CountDownLatch countDownLatch = new CountDownLatch(1); ActionListener completionListener = new LatchedActionListener<>(new PlainActionFuture<>(), countDownLatch); - ReadContextListener readContextListener = new ReadContextListener(TEST_SEGMENT_FILE, fileLocation, threadPool, completionListener); + ReadContextListener readContextListener = new ReadContextListener( + TEST_SEGMENT_FILE, + fileLocation, + completionListener, + threadPool, + UnaryOperator.identity(), + MAX_CONCURRENT_STREAMS + ); InputStream badInputStream = new InputStream() { @Override @@ -101,7 +118,13 @@ public int available() { } }; - blobPartStreams.add(NUMBER_OF_PARTS, new InputStreamContainer(badInputStream, PART_SIZE, PART_SIZE * NUMBER_OF_PARTS)); + blobPartStreams.add( + NUMBER_OF_PARTS, + () -> CompletableFuture.supplyAsync( + () -> new InputStreamContainer(badInputStream, PART_SIZE, PART_SIZE * NUMBER_OF_PARTS), + threadPool.generic() + ) + ); ReadContext readContext = new ReadContext((long) (PART_SIZE + 1) * NUMBER_OF_PARTS, blobPartStreams, null); readContextListener.onResponse(readContext); @@ -112,18 +135,31 @@ public int available() { public void testReadContextListenerException() { Path fileLocation = path.resolve(UUID.randomUUID().toString()); CountingCompletionListener listener = new CountingCompletionListener(); - ReadContextListener readContextListener = new ReadContextListener(TEST_SEGMENT_FILE, fileLocation, threadPool, listener); + ReadContextListener readContextListener = new ReadContextListener( + TEST_SEGMENT_FILE, + fileLocation, + listener, + threadPool, + UnaryOperator.identity(), + MAX_CONCURRENT_STREAMS + ); IOException exception = new IOException(); readContextListener.onFailure(exception); assertEquals(1, listener.getFailureCount()); assertEquals(exception, listener.getException()); } - private List initializeBlobPartStreams() { - List blobPartStreams = new ArrayList<>(); + private List initializeBlobPartStreams() { + List blobPartStreams = new ArrayList<>(); for (int partNumber = 0; partNumber < NUMBER_OF_PARTS; partNumber++) { InputStream testStream = new ByteArrayInputStream(randomByteArrayOfLength(PART_SIZE)); - blobPartStreams.add(new InputStreamContainer(testStream, PART_SIZE, (long) partNumber * PART_SIZE)); + int finalPartNumber = partNumber; + blobPartStreams.add( + () -> CompletableFuture.supplyAsync( + () -> new InputStreamContainer(testStream, PART_SIZE, (long) finalPartNumber * PART_SIZE), + threadPool.generic() + ) + ); } return blobPartStreams; } diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index a1d6be84c9926..bbd73bcf97aab 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -105,6 +105,7 @@ import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; +import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; @@ -257,7 +258,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { writableRegistry(), () -> false, null, - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool, DefaultRecoverySettings.INSTANCE), translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 5a13f57db2c87..941f2f48e71af 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -33,6 +33,7 @@ import org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils; import org.opensearch.index.store.Store; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; +import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.threadpool.ThreadPool; @@ -155,7 +156,8 @@ public void testRemoteDirectoryInitThrowsException() throws IOException { remoteMetadataDirectory, mock(RemoteStoreLockManager.class), mock(ThreadPool.class), - shardId + shardId, + DefaultRecoverySettings.INSTANCE ); FilterDirectory remoteStoreFilterDirectory = new RemoteStoreRefreshListenerTests.TestFilterDirectory( new RemoteStoreRefreshListenerTests.TestFilterDirectory(remoteSegmentStoreDirectory) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index cad5e47531cc6..78c7fe64cebd9 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -20,6 +20,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; +import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -57,7 +58,11 @@ public void setup() { repositoriesService = mock(RepositoriesService.class); threadPool = mock(ThreadPool.class); when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); - remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(repositoriesServiceSupplier, threadPool); + remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( + repositoriesServiceSupplier, + threadPool, + DefaultRecoverySettings.INSTANCE + ); } public void testNewDirectory() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index f154dddb0e7cc..b574ccaac55e1 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -25,12 +25,13 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.common.blobstore.stream.write.WriteContext; +import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; @@ -41,6 +42,7 @@ import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; +import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.threadpool.ThreadPool; import org.junit.After; @@ -56,9 +58,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.UnaryOperator; import org.mockito.Mockito; @@ -145,13 +149,16 @@ public void setup() throws IOException { remoteMetadataDirectory, mdLockManager, threadPool, - indexShard.shardId() + indexShard.shardId(), + DefaultRecoverySettings.INSTANCE ); try (Store store = indexShard.store()) { segmentInfos = store.readLastCommittedSegmentsInfo(); } + when(remoteDataDirectory.getDownloadRateLimiter()).thenReturn(UnaryOperator.identity()); when(threadPool.executor(ThreadPool.Names.REMOTE_PURGE)).thenReturn(executorService); + when(threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY)).thenReturn(executorService); } @After @@ -562,9 +569,6 @@ public void onFailure(Exception e) {} } public void testCopyFilesToMultipart() throws Exception { - Settings settings = Settings.builder().build(); - FeatureFlags.initializeFeatureFlags(settings); - String filename = "_0.cfe"; populateMetadata(); remoteSegmentStoreDirectory.init(); @@ -574,13 +578,15 @@ public void testCopyFilesToMultipart() throws Exception { when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); Mockito.doAnswer(invocation -> { - ActionListener completionListener = invocation.getArgument(3); - completionListener.onResponse(invocation.getArgument(0)); + ActionListener completionListener = invocation.getArgument(1); + final CompletableFuture future = new CompletableFuture<>(); + future.complete(new InputStreamContainer(new ByteArrayInputStream(new byte[] { 42 }), 0, 1)); + completionListener.onResponse(new ReadContext(1, List.of(() -> future), "")); return null; - }).when(blobContainer).asyncBlobDownload(any(), any(), any(), any()); + }).when(blobContainer).readBlobAsync(any(), any()); CountDownLatch downloadLatch = new CountDownLatch(1); - ActionListener completionListener = new ActionListener() { + ActionListener completionListener = new ActionListener<>() { @Override public void onResponse(String unused) { downloadLatch.countDown(); @@ -592,7 +598,7 @@ public void onFailure(Exception e) {} Path path = createTempDir(); remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, completionListener); assertTrue(downloadLatch.await(5000, TimeUnit.SECONDS)); - verify(blobContainer, times(1)).asyncBlobDownload(contains(filename), eq(path.resolve(filename)), any(), any()); + verify(blobContainer, times(1)).readBlobAsync(contains(filename), any()); verify(storeDirectory, times(0)).copyFrom(any(), any(), any(), any()); } @@ -678,7 +684,8 @@ public void testCopyFilesFromMultipartIOException() throws Exception { remoteMetadataDirectory, mdLockManager, threadPool, - indexShard.shardId() + indexShard.shardId(), + DefaultRecoverySettings.INSTANCE ); populateMetadata(); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 09f5c1bea1a5e..80731b378f369 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -195,6 +195,7 @@ import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.indices.mapper.MapperRegistry; +import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.indices.recovery.PeerRecoverySourceService; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoverySettings; @@ -2066,7 +2067,7 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool, DefaultRecoverySettings.INSTANCE), repositoriesServiceReference::get, fileCacheCleaner, null, diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 466c00d0648dc..186c1c7e78f6b 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -119,6 +119,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.indices.recovery.AsyncRecoveryTarget; +import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoveryFailedException; import org.opensearch.indices.recovery.RecoveryResponse; @@ -640,7 +641,7 @@ protected IndexShard newShard( Collections.emptyList(), clusterSettings ); - Store remoteStore = null; + Store remoteStore; RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory = null; RepositoriesService mockRepoSvc = mock(RepositoriesService.class); @@ -659,6 +660,8 @@ protected IndexShard newShard( remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, indexSettings.getSettings()); BlobStoreRepository repo = createRepository(remotePath); when(mockRepoSvc.repository(any())).thenAnswer(invocationOnMock -> repo); + } else { + remoteStore = null; } final BiFunction translogFactorySupplier = (settings, shardRouting) -> { @@ -698,7 +701,8 @@ protected IndexShard newShard( remoteStore, remoteStoreStatsTrackerFactory, () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, - "dummy-node" + "dummy-node", + null ); indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER); if (remoteStoreStatsTrackerFactory != null) { @@ -785,7 +789,14 @@ protected RemoteSegmentStoreDirectory createRemoteSegmentStoreDirectory(ShardId RemoteStoreLockManager remoteStoreLockManager = new RemoteStoreMetadataLockManager( new RemoteBufferedOutputDirectory(getBlobContainer(remoteShardPath.resolveIndex())) ); - return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, remoteStoreLockManager, threadPool, shardId); + return new RemoteSegmentStoreDirectory( + dataDirectory, + metadataDirectory, + remoteStoreLockManager, + threadPool, + shardId, + DefaultRecoverySettings.INSTANCE + ); } private RemoteDirectory newRemoteDirectory(Path f) throws IOException { diff --git a/test/framework/src/main/java/org/opensearch/indices/recovery/DefaultRecoverySettings.java b/test/framework/src/main/java/org/opensearch/indices/recovery/DefaultRecoverySettings.java new file mode 100644 index 0000000000000..359668f5dad71 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/indices/recovery/DefaultRecoverySettings.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.recovery; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; + +/** + * Utility to provide a {@link RecoverySettings} instance containing all defaults + */ +public final class DefaultRecoverySettings { + private DefaultRecoverySettings() {} + + public static final RecoverySettings INSTANCE = new RecoverySettings( + Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); +} From de8511b27487ec90a1632f59ad91940374c9c862 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 4 Oct 2023 18:36:07 -0500 Subject: [PATCH 104/388] Mute DeleteSnapshotIT.testDeleteShallowCopySnapshot (#10372) Signed-off-by: Andrew Ross --- .../java/org/opensearch/snapshots/DeleteSnapshotIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index e79bf1c16b586..31abc16bba50e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -64,6 +64,7 @@ public void testDeleteSnapshot() throws Exception { assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9115") public void testDeleteShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); From 2965e69aff83b702d46d1d630998cbf3ef7ebca5 Mon Sep 17 00:00:00 2001 From: Prabhat <20185657+CaptainDredge@users.noreply.github.com> Date: Wed, 4 Oct 2023 17:17:32 -0700 Subject: [PATCH 105/388] Added performance improvement for datetime field parsing (#9567) * Added performance improvement for datetime field parsing This adds caching of formatters in case of no explicit format specified for the datetime field in mapping. This also adds `strict_date_time_no_millis` as additional formatter in default date time formats Signed-off-by: Prabhat Sharma * Refactor DateTimeFormatter Access under featireflag Signed-off-by: Prabhat Sharma --------- Signed-off-by: Prabhat Sharma Co-authored-by: Prabhat Sharma --- CHANGELOG.md | 1 + distribution/src/config/opensearch.yml | 6 + .../aggregations/bucket/DateHistogramIT.java | 6 +- .../bucket/DateHistogramOffsetIT.java | 2 +- .../common/joda/JodaDateFormatter.java | 5 + .../common/settings/FeatureFlagSettings.java | 3 +- .../opensearch/common/time/DateFormatter.java | 33 +++++- .../common/time/JavaDateFormatter.java | 105 +++++++++++++++--- .../opensearch/common/util/FeatureFlags.java | 22 ++++ .../index/mapper/DateFieldMapper.java | 40 ++++++- .../index/mapper/RangeFieldMapper.java | 2 +- .../opensearch/index/mapper/RangeType.java | 2 +- .../index/mapper/RootObjectMapper.java | 2 +- .../opensearch/script/ScoreScriptUtils.java | 5 +- .../org/opensearch/search/DocValueFormat.java | 16 ++- .../support/CoreValuesSourceType.java | 2 +- .../aggregations/support/ValueType.java | 2 +- .../BottomSortValuesCollectorTests.java | 17 ++- .../index/mapper/DateFieldMapperTests.java | 3 +- .../index/mapper/DateFieldTypeTests.java | 22 ++-- .../index/mapper/RangeFieldMapperTests.java | 7 +- .../index/mapper/RangeFieldTypeTests.java | 4 +- .../index/query/RangeQueryBuilderTests.java | 4 +- .../aggregations/AggregatorBaseTests.java | 4 +- .../DateHistogramAggregatorTestCase.java | 2 +- .../DateHistogramAggregatorTests.java | 2 +- .../DateRangeHistogramAggregatorTests.java | 2 +- .../range/DateRangeAggregatorTests.java | 2 +- .../bucket/range/RangeAggregatorTests.java | 4 +- .../pipeline/AvgBucketAggregatorTests.java | 2 +- .../CumulativeSumAggregatorTests.java | 2 +- .../pipeline/MovFnAggrgatorTests.java | 2 +- .../BaseCompositeAggregatorTestCase.java | 2 +- 33 files changed, 265 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50922b85a0c0d..dea5989d6d718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855)) +- Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558)) ### Deprecated diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 1d2cfe7eccae6..b7ab2e1c2309b 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -121,3 +121,9 @@ ${path.logs} # index searcher threadpool. # #opensearch.experimental.feature.concurrent_segment_search.enabled: false +# +# +# Gates the optimization of datetime formatters caching along with change in default datetime formatter +# Once there is no observed impact on performance, this feature flag can be removed. +# +#opensearch.experimental.optimization.datetime_formatter_caching.enabled: false diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java index 4ce8af3e0f081..895e24ba65132 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java @@ -124,7 +124,7 @@ protected Settings featureFlagSettings() { } private ZonedDateTime date(String date) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date)); } private static String format(ZonedDateTime date, String pattern) { @@ -1624,8 +1624,8 @@ public void testScriptCaching() throws Exception { .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) .get() ); - String date = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(date(1, 1)); - String date2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(date(2, 1)); + String date = DateFieldMapper.getDefaultDateTimeFormatter().format(date(1, 1)); + String date2 = DateFieldMapper.getDefaultDateTimeFormatter().format(date(2, 1)); indexRandom( true, client().prepareIndex("cache_test_idx").setId("1").setSource("d", date), diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 04115f69172da..d44071e1ef9c5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -92,7 +92,7 @@ protected Settings featureFlagSettings() { } private ZonedDateTime date(String date) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date)); } @Before diff --git a/server/src/main/java/org/opensearch/common/joda/JodaDateFormatter.java b/server/src/main/java/org/opensearch/common/joda/JodaDateFormatter.java index 12d48a0b362ce..bf25e5b1b3923 100644 --- a/server/src/main/java/org/opensearch/common/joda/JodaDateFormatter.java +++ b/server/src/main/java/org/opensearch/common/joda/JodaDateFormatter.java @@ -125,6 +125,11 @@ public String pattern() { return pattern; } + @Override + public String printPattern() { + throw new UnsupportedOperationException("JodaDateFormatter does not have a print pattern"); + } + @Override public Locale locale() { return printer.getLocale(); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 90abc0a0765c1..387b0c9753574 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -39,7 +39,8 @@ protected FeatureFlagSettings( FeatureFlags.EXTENSIONS_SETTING, FeatureFlags.IDENTITY_SETTING, FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING, - FeatureFlags.TELEMETRY_SETTING + FeatureFlags.TELEMETRY_SETTING, + FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/time/DateFormatter.java b/server/src/main/java/org/opensearch/common/time/DateFormatter.java index d57fd441b9bf4..c98bd853dfced 100644 --- a/server/src/main/java/org/opensearch/common/time/DateFormatter.java +++ b/server/src/main/java/org/opensearch/common/time/DateFormatter.java @@ -126,6 +126,14 @@ default String formatJoda(DateTime dateTime) { */ String pattern(); + /** + * A name based format for this formatter. Can be one of the registered formatters like epoch_millis or + * a configured format like HH:mm:ss + * + * @return The name of this formatter + */ + String printPattern(); + /** * Returns the configured locale of the date formatter * @@ -147,7 +155,7 @@ default String formatJoda(DateTime dateTime) { */ DateMathParser toDateMathParser(); - static DateFormatter forPattern(String input) { + static DateFormatter forPattern(String input, String printPattern, Boolean canCacheFormatter) { if (Strings.hasLength(input) == false) { throw new IllegalArgumentException("No date pattern provided"); @@ -158,7 +166,28 @@ static DateFormatter forPattern(String input) { List patterns = splitCombinedPatterns(format); List formatters = patterns.stream().map(DateFormatters::forPattern).collect(Collectors.toList()); - return JavaDateFormatter.combined(input, formatters); + DateFormatter printFormatter = formatters.get(0); + if (Strings.hasLength(printPattern)) { + String printFormat = strip8Prefix(printPattern); + try { + printFormatter = DateFormatters.forPattern(printFormat); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid print format: " + e.getMessage(), e); + } + } + return JavaDateFormatter.combined(input, formatters, printFormatter, canCacheFormatter); + } + + static DateFormatter forPattern(String input) { + return forPattern(input, null, false); + } + + static DateFormatter forPattern(String input, String printPattern) { + return forPattern(input, printPattern, false); + } + + static DateFormatter forPattern(String input, Boolean canCacheFormatter) { + return forPattern(input, null, canCacheFormatter); } static String strip8Prefix(String input) { diff --git a/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java b/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java index 07013a3dc75f2..594ebc3b26a73 100644 --- a/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java +++ b/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java @@ -32,6 +32,7 @@ package org.opensearch.common.time; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import java.text.ParsePosition; @@ -51,6 +52,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiConsumer; import java.util.stream.Collectors; @@ -67,9 +69,11 @@ class JavaDateFormatter implements DateFormatter { } private final String format; + private final String printFormat; private final DateTimeFormatter printer; private final List parsers; private final JavaDateFormatter roundupParser; + private final Boolean canCacheLastParsedFormatter; /** * A round up formatter @@ -93,8 +97,18 @@ JavaDateFormatter getRoundupParser() { } // named formatters use default roundUpParser + JavaDateFormatter( + String format, + String printFormat, + DateTimeFormatter printer, + Boolean canCacheLastParsedFormatter, + DateTimeFormatter... parsers + ) { + this(format, printFormat, printer, ROUND_UP_BASE_FIELDS, canCacheLastParsedFormatter, parsers); + } + JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter... parsers) { - this(format, printer, ROUND_UP_BASE_FIELDS, parsers); + this(format, format, printer, false, parsers); } private static final BiConsumer ROUND_UP_BASE_FIELDS = (builder, parser) -> { @@ -111,8 +125,10 @@ JavaDateFormatter getRoundupParser() { // subclasses override roundUpParser JavaDateFormatter( String format, + String printFormat, DateTimeFormatter printer, BiConsumer roundupParserConsumer, + Boolean canCacheLastParsedFormatter, DateTimeFormatter... parsers ) { if (printer == null) { @@ -128,16 +144,27 @@ JavaDateFormatter getRoundupParser() { } this.printer = printer; this.format = format; + this.printFormat = printFormat; + this.canCacheLastParsedFormatter = canCacheLastParsedFormatter; if (parsers.length == 0) { this.parsers = Collections.singletonList(printer); } else { - this.parsers = Arrays.asList(parsers); + this.parsers = new CopyOnWriteArrayList<>(parsers); } List roundUp = createRoundUpParser(format, roundupParserConsumer); this.roundupParser = new RoundUpFormatter(format, roundUp); } + JavaDateFormatter( + String format, + DateTimeFormatter printer, + BiConsumer roundupParserConsumer, + DateTimeFormatter... parsers + ) { + this(format, format, printer, roundupParserConsumer, false, parsers); + } + /** * This is when the RoundUp Formatters are created. In further merges (with ||) it will only append them to a list. * || is not expected to be provided as format when a RoundUp formatter is created. It will be splitted before in @@ -164,36 +191,61 @@ private List createRoundUpParser( return null; } - public static DateFormatter combined(String input, List formatters) { + public static DateFormatter combined( + String input, + List formatters, + DateFormatter printFormatter, + Boolean canCacheLastParsedFormatter + ) { assert formatters.size() > 0; + assert printFormatter != null; List parsers = new ArrayList<>(formatters.size()); List roundUpParsers = new ArrayList<>(formatters.size()); - DateTimeFormatter printer = null; + assert printFormatter instanceof JavaDateFormatter; + JavaDateFormatter javaPrintFormatter = (JavaDateFormatter) printFormatter; + DateTimeFormatter printer = javaPrintFormatter.getPrinter(); for (DateFormatter formatter : formatters) { assert formatter instanceof JavaDateFormatter; JavaDateFormatter javaDateFormatter = (JavaDateFormatter) formatter; - if (printer == null) { - printer = javaDateFormatter.getPrinter(); - } parsers.addAll(javaDateFormatter.getParsers()); roundUpParsers.addAll(javaDateFormatter.getRoundupParser().getParsers()); } - return new JavaDateFormatter(input, printer, roundUpParsers, parsers); + return new JavaDateFormatter( + input, + javaPrintFormatter.format, + printer, + roundUpParsers, + parsers, + canCacheLastParsedFormatter & FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING) + ); // check if caching is enabled } private JavaDateFormatter( String format, + String printFormat, DateTimeFormatter printer, List roundUpParsers, - List parsers + List parsers, + Boolean canCacheLastParsedFormatter ) { this.format = format; + this.printFormat = printFormat; this.printer = printer; this.roundupParser = roundUpParsers != null ? new RoundUpFormatter(format, roundUpParsers) : null; - this.parsers = parsers; + this.parsers = new CopyOnWriteArrayList<>(parsers); + this.canCacheLastParsedFormatter = canCacheLastParsedFormatter; + } + + private JavaDateFormatter( + String format, + DateTimeFormatter printer, + List roundUpParsers, + List parsers + ) { + this(format, format, printer, roundUpParsers, parsers, false); } JavaDateFormatter getRoundupParser() { @@ -233,12 +285,24 @@ public TemporalAccessor parse(String input) { */ private TemporalAccessor doParse(String input) { if (parsers.size() > 1) { + Object object = null; + DateTimeFormatter lastParsedformatter = null; for (DateTimeFormatter formatter : parsers) { ParsePosition pos = new ParsePosition(0); - Object object = formatter.toFormat().parseObject(input, pos); + object = formatter.toFormat().parseObject(input, pos); if (parsingSucceeded(object, input, pos)) { - return (TemporalAccessor) object; + lastParsedformatter = formatter; + break; + } + } + if (lastParsedformatter != null) { + if (canCacheLastParsedFormatter && lastParsedformatter != parsers.get(0)) { + synchronized (parsers) { + parsers.remove(lastParsedformatter); + parsers.add(0, lastParsedformatter); + } } + return (TemporalAccessor) object; } throw new DateTimeParseException("Failed to parse with all enclosed parsers", input, 0); } @@ -255,12 +319,14 @@ public DateFormatter withZone(ZoneId zoneId) { if (zoneId.equals(zone())) { return this; } - List parsers = this.parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList()); + List parsers = new CopyOnWriteArrayList<>( + this.parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList()) + ); List roundUpParsers = this.roundupParser.getParsers() .stream() .map(p -> p.withZone(zoneId)) .collect(Collectors.toList()); - return new JavaDateFormatter(format, printer.withZone(zoneId), roundUpParsers, parsers); + return new JavaDateFormatter(format, printFormat, printer.withZone(zoneId), roundUpParsers, parsers, canCacheLastParsedFormatter); } @Override @@ -269,12 +335,14 @@ public DateFormatter withLocale(Locale locale) { if (locale.equals(locale())) { return this; } - List parsers = this.parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList()); + List parsers = new CopyOnWriteArrayList<>( + this.parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList()) + ); List roundUpParsers = this.roundupParser.getParsers() .stream() .map(p -> p.withLocale(locale)) .collect(Collectors.toList()); - return new JavaDateFormatter(format, printer.withLocale(locale), roundUpParsers, parsers); + return new JavaDateFormatter(format, printFormat, printer.withLocale(locale), roundUpParsers, parsers, canCacheLastParsedFormatter); } @Override @@ -287,6 +355,11 @@ public String pattern() { return format; } + @Override + public String printPattern() { + return printFormat; + } + @Override public Locale locale() { return this.printer.getLocale(); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index b89d2d0549823..4e9b417e3433b 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -55,6 +55,11 @@ public class FeatureFlags { */ public static final String TELEMETRY = "opensearch.experimental.feature.telemetry.enabled"; + /** + * Gates the optimization of datetime formatters caching along with change in default datetime formatter. + */ + public static final String DATETIME_FORMATTER_CACHING = "opensearch.experimental.optimization.datetime_formatter_caching.enabled"; + /** * Should store the settings from opensearch.yml. */ @@ -83,6 +88,17 @@ public static boolean isEnabled(String featureFlagName) { return settings != null && settings.getAsBoolean(featureFlagName, false); } + public static boolean isEnabled(Setting featureFlag) { + if ("true".equalsIgnoreCase(System.getProperty(featureFlag.getKey()))) { + // TODO: Remove the if condition once FeatureFlags are only supported via opensearch.yml + return true; + } else if (settings != null) { + return featureFlag.get(settings); + } else { + return featureFlag.getDefault(Settings.EMPTY); + } + } + public static final Setting SEGMENT_REPLICATION_EXPERIMENTAL_SETTING = Setting.boolSetting( SEGMENT_REPLICATION_EXPERIMENTAL, false, @@ -100,4 +116,10 @@ public static boolean isEnabled(String featureFlagName) { false, Property.NodeScope ); + + public static final Setting DATETIME_FORMATTER_CACHING_SETTING = Setting.boolSetting( + DATETIME_FORMATTER_CACHING, + true, + Property.NodeScope + ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index 393ddf2dd1de0..3b832628695fe 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -52,6 +52,7 @@ import org.opensearch.common.time.DateMathParser; import org.opensearch.common.time.DateUtils; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.LocaleUtils; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; @@ -91,7 +92,21 @@ public final class DateFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "date"; public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; - public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); + @Deprecated + public static final DateFormatter LEGACY_DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern( + // TODO remove in 3.0 after backporting + "strict_date_optional_time||epoch_millis" + ); + public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern( + "strict_date_time_no_millis||strict_date_optional_time||epoch_millis", + "strict_date_optional_time" + ); + + public static DateFormatter getDefaultDateTimeFormatter() { + return FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING) + ? DEFAULT_DATE_TIME_FORMATTER + : LEGACY_DEFAULT_DATE_TIME_FORMATTER; + } /** * Resolution of the date time @@ -223,8 +238,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder { "format", false, m -> toType(m).format, - DEFAULT_DATE_TIME_FORMATTER.pattern() + getDefaultDateTimeFormatter().pattern() ); + private final Parameter printFormat = Parameter.stringParam( + "print_format", + false, + m -> toType(m).printFormat, + getDefaultDateTimeFormatter().printPattern() + ).acceptsNull(); private final Parameter locale = new Parameter<>( "locale", false, @@ -253,13 +274,18 @@ public Builder( this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault); if (dateFormatter != null) { this.format.setValue(dateFormatter.pattern()); + this.printFormat.setValue(dateFormatter.printPattern()); this.locale.setValue(dateFormatter.locale()); } } private DateFormatter buildFormatter() { try { - return DateFormatter.forPattern(format.getValue()).withLocale(locale.getValue()); + if (format.isConfigured() && !printFormat.isConfigured()) { + return DateFormatter.forPattern(format.getValue(), null, !format.isConfigured()).withLocale(locale.getValue()); + } + return DateFormatter.forPattern(format.getValue(), printFormat.getValue(), !format.isConfigured()) + .withLocale(locale.getValue()); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e); } @@ -267,7 +293,7 @@ private DateFormatter buildFormatter() { @Override protected List> getParameters() { - return Arrays.asList(index, docValues, store, format, locale, nullValue, ignoreMalformed, boost, meta); + return Arrays.asList(index, docValues, store, format, printFormat, locale, nullValue, ignoreMalformed, boost, meta); } private Long parseNullValue(DateFieldType fieldType) { @@ -346,7 +372,7 @@ public DateFieldType( } public DateFieldType(String name) { - this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, Collections.emptyMap()); + this(name, true, false, true, getDefaultDateTimeFormatter(), Resolution.MILLISECONDS, null, Collections.emptyMap()); } public DateFieldType(String name, DateFormatter dateFormatter) { @@ -354,7 +380,7 @@ public DateFieldType(String name, DateFormatter dateFormatter) { } public DateFieldType(String name, Resolution resolution) { - this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap()); + this(name, true, false, true, getDefaultDateTimeFormatter(), resolution, null, Collections.emptyMap()); } public DateFieldType(String name, Resolution resolution, DateFormatter dateFormatter) { @@ -610,6 +636,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final boolean hasDocValues; private final Locale locale; private final String format; + private final String printFormat; private final boolean ignoreMalformed; private final Long nullValue; private final String nullValueAsString; @@ -633,6 +660,7 @@ private DateFieldMapper( this.hasDocValues = builder.docValues.getValue(); this.locale = builder.locale.getValue(); this.format = builder.format.getValue(); + this.printFormat = builder.printFormat.getValue(); this.ignoreMalformed = builder.ignoreMalformed.getValue(); this.nullValueAsString = builder.nullValue.getValue(); this.nullValue = nullValue; diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java index 10f179c964591..05ca7dee0fe4b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java @@ -95,7 +95,7 @@ public class RangeFieldMapper extends ParametrizedFieldMapper { */ public static class Defaults { public static final Explicit COERCE = new Explicit<>(true, false); - public static final DateFormatter DATE_FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; + public static final DateFormatter DATE_FORMATTER = DateFieldMapper.getDefaultDateTimeFormatter(); } // this is private since it has a different default diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeType.java b/server/src/main/java/org/opensearch/index/mapper/RangeType.java index c8cd317779c7c..7e29fd417845b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeType.java @@ -313,7 +313,7 @@ public Query rangeQuery( ) { ZoneId zone = (timeZone == null) ? ZoneOffset.UTC : timeZone; - DateMathParser dateMathParser = (parser == null) ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser() : parser; + DateMathParser dateMathParser = (parser == null) ? DateFieldMapper.getDefaultDateTimeFormatter().toDateMathParser() : parser; boolean roundUp = includeLower == false; // using "gt" should round lower bound up Long low = lowerTerm == null ? minValue() diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index ce3d6eef19af2..9d1ccc9e795ba 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -72,7 +72,7 @@ public class RootObjectMapper extends ObjectMapper { */ public static class Defaults { public static final DateFormatter[] DYNAMIC_DATE_TIME_FORMATTERS = new DateFormatter[] { - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), DateFormatter.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis") }; public static final boolean DATE_DETECTION = true; public static final boolean NUMERIC_DETECTION = false; diff --git a/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java b/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java index 378ff1ccfe4a8..2396aa369e98d 100644 --- a/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java +++ b/server/src/main/java/org/opensearch/script/ScoreScriptUtils.java @@ -343,11 +343,14 @@ public double decayNumericGauss(double docValue) { /** * Limitations: since script functions don't have access to DateFieldMapper, * decay functions on dates are limited to dates in the default format and default time zone, + * Further, since script module gets initialized before the featureflags are loaded, + * we cannot use the feature flag to gate the usage of the new default date format. * Also, using calculations with now are not allowed. * */ private static final ZoneId defaultZoneId = ZoneId.of("UTC"); - private static final DateMathParser dateParser = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser(); + // ToDo: use new default date formatter once feature flag is removed + private static final DateMathParser dateParser = DateFieldMapper.LEGACY_DEFAULT_DATE_TIME_FORMATTER.toDateMathParser(); /** * Linear date decay diff --git a/server/src/main/java/org/opensearch/search/DocValueFormat.java b/server/src/main/java/org/opensearch/search/DocValueFormat.java index c6187e5949035..412191c57abd8 100644 --- a/server/src/main/java/org/opensearch/search/DocValueFormat.java +++ b/server/src/main/java/org/opensearch/search/DocValueFormat.java @@ -243,7 +243,12 @@ public DateTime(DateFormatter formatter, ZoneId timeZone, DateFieldMapper.Resolu } public DateTime(StreamInput in) throws IOException { - this.formatter = DateFormatter.forPattern(in.readString()); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.formatter = DateFormatter.forPattern(in.readString(), in.readOptionalString()); + } else { + this.formatter = DateFormatter.forPattern(in.readString()); + } + this.parser = formatter.toDateMathParser(); String zoneId = in.readString(); this.timeZone = ZoneId.of(zoneId); @@ -260,7 +265,14 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(formatter.pattern()); + if (out.getVersion().before(Version.V_3_0_0) && formatter.equals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER)) { + out.writeString(DateFieldMapper.LEGACY_DEFAULT_DATE_TIME_FORMATTER.pattern()); // required for backwards compatibility + } else { + out.writeString(formatter.pattern()); + } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalString(formatter.printPattern()); + } out.writeString(timeZone.getId()); out.writeVInt(resolution.ordinal()); if (out.getVersion().before(Version.V_3_0_0)) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java index 224f9281705e1..00c769c015fbc 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java @@ -411,7 +411,7 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, @Override public DocValueFormat getFormatter(String format, ZoneId tz) { return new DocValueFormat.DateTime( - format == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER : DateFormatter.forPattern(format), + format == null ? DateFieldMapper.getDefaultDateTimeFormatter() : DateFormatter.forPattern(format), tz == null ? ZoneOffset.UTC : tz, // If we were just looking at fields, we could read the resolution from the field settings, but we need to deal with script // output, which has no way to indicate the resolution, so we need to default to something. Milliseconds is the standard. diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java index 59fa2e03f0bc3..33fefa57d50f0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValueType.java @@ -61,7 +61,7 @@ public enum ValueType implements Writeable { "date", "date", CoreValuesSourceType.DATE, - new DocValueFormat.DateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ZoneOffset.UTC, DateFieldMapper.Resolution.MILLISECONDS) + new DocValueFormat.DateTime(DateFieldMapper.getDefaultDateTimeFormatter(), ZoneOffset.UTC, DateFieldMapper.Resolution.MILLISECONDS) ), IP((byte) 6, "ip", "ip", CoreValuesSourceType.IP, DocValueFormat.IP), // TODO: what is the difference between "number" and "numeric"? diff --git a/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java b/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java index 3efcadbfb320d..8042a7e296869 100644 --- a/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java +++ b/server/src/test/java/org/opensearch/action/search/BottomSortValuesCollectorTests.java @@ -46,7 +46,6 @@ import java.time.ZoneId; import java.util.Arrays; -import static org.opensearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO; @@ -136,7 +135,11 @@ public void testWithDates() { for (boolean reverse : new boolean[] { true, false }) { SortField[] sortFields = new SortField[] { new SortField("foo", SortField.Type.LONG, reverse) }; DocValueFormat[] sortFormats = new DocValueFormat[] { - new DocValueFormat.DateTime(DEFAULT_DATE_TIME_FORMATTER, ZoneId.of("UTC"), DateFieldMapper.Resolution.MILLISECONDS) }; + new DocValueFormat.DateTime( + DateFieldMapper.getDefaultDateTimeFormatter(), + ZoneId.of("UTC"), + DateFieldMapper.Resolution.MILLISECONDS + ) }; BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields); collector.consumeTopDocs( createTopDocs(sortFields[0], 100, newDateArray("2017-06-01T12:18:20Z", "2018-04-03T15:10:27Z", "2013-06-01T13:10:20Z")), @@ -170,7 +173,11 @@ public void testWithDateNanos() { for (boolean reverse : new boolean[] { true, false }) { SortField[] sortFields = new SortField[] { new SortField("foo", SortField.Type.LONG, reverse) }; DocValueFormat[] sortFormats = new DocValueFormat[] { - new DocValueFormat.DateTime(DEFAULT_DATE_TIME_FORMATTER, ZoneId.of("UTC"), DateFieldMapper.Resolution.NANOSECONDS) }; + new DocValueFormat.DateTime( + DateFieldMapper.getDefaultDateTimeFormatter(), + ZoneId.of("UTC"), + DateFieldMapper.Resolution.NANOSECONDS + ) }; BottomSortValuesCollector collector = new BottomSortValuesCollector(3, sortFields); collector.consumeTopDocs( createTopDocs(sortFields[0], 100, newDateNanoArray("2017-06-01T12:18:20Z", "2018-04-03T15:10:27Z", "2013-06-01T13:10:20Z")), @@ -242,7 +249,7 @@ private Object[] newBytesArray(String... values) { private Object[] newDateArray(String... values) { Long[] longs = new Long[values.length]; for (int i = 0; i < values.length; i++) { - longs[i] = DEFAULT_DATE_TIME_FORMATTER.parseMillis(values[i]); + longs[i] = DateFieldMapper.getDefaultDateTimeFormatter().parseMillis(values[i]); } return longs; } @@ -250,7 +257,7 @@ private Object[] newDateArray(String... values) { private Object[] newDateNanoArray(String... values) { Long[] longs = new Long[values.length]; for (int i = 0; i < values.length; i++) { - longs[i] = DateUtils.toNanoSeconds(DEFAULT_DATE_TIME_FORMATTER.parseMillis(values[i])); + longs[i] = DateUtils.toNanoSeconds(DateFieldMapper.getDefaultDateTimeFormatter().parseMillis(values[i])); } return longs; } diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java index 2afd6773b15d4..054d3956596af 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java @@ -66,6 +66,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("index", b -> b.field("index", false)); checker.registerConflictCheck("store", b -> b.field("store", true)); checker.registerConflictCheck("format", b -> b.field("format", "yyyy-MM-dd")); + checker.registerConflictCheck("print_format", b -> b.field("print_format", "yyyy-MM-dd")); checker.registerConflictCheck("locale", b -> b.field("locale", "es")); checker.registerConflictCheck("null_value", b -> b.field("null_value", "34500000")); checker.registerUpdateCheck(b -> b.field("ignore_malformed", true), m -> assertTrue(((DateFieldMapper) m).getIgnoreMalformed())); @@ -148,7 +149,7 @@ public void testStore() throws Exception { public void testIgnoreMalformed() throws IOException { testIgnoreMalformedForValue( "2016-03-99", - "failed to parse date field [2016-03-99] with format [strict_date_optional_time||epoch_millis]" + "failed to parse date field [2016-03-99] with format [strict_date_time_no_millis||strict_date_optional_time||epoch_millis]" ); testIgnoreMalformedForValue("-2147483648", "Invalid value for Year (valid values -999999999 - 999999999): -2147483648"); testIgnoreMalformedForValue("-522000000", "long overflow"); diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index 37d4a53f36878..ab53ae81ab0ce 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -107,7 +107,7 @@ public void isFieldWithinRangeTestCase(DateFieldType ft) throws IOException { w.addDocument(doc); DirectoryReader reader = DirectoryReader.open(w); - DateMathParser alternateFormat = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser(); + DateMathParser alternateFormat = DateFieldMapper.getDefaultDateTimeFormatter().toDateMathParser(); doTestIsFieldWithinQuery(ft, reader, null, null); doTestIsFieldWithinQuery(ft, reader, null, alternateFormat); doTestIsFieldWithinQuery(ft, reader, DateTimeZone.UTC, null); @@ -158,7 +158,7 @@ private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader, public void testValueFormat() { MappedFieldType ft = new DateFieldType("field"); - long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12T14:10:55")) + long instant = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse("2015-10-12T14:10:55")) .toInstant() .toEpochMilli(); @@ -167,14 +167,14 @@ public void testValueFormat() { assertEquals("2015", new DateFieldType("field").docValueFormat("YYYY", ZoneOffset.UTC).format(instant)); assertEquals(instant, ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12T14:10:55", false, null)); assertEquals(instant + 999, ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12T14:10:55", true, null)); - long i = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-13")).toInstant().toEpochMilli(); + long i = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse("2015-10-13")).toInstant().toEpochMilli(); assertEquals(i - 1, ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12||/d", true, null)); } public void testValueForSearch() { MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T12:09:55.000Z"; - long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(date); + long instant = DateFieldMapper.getDefaultDateTimeFormatter().parseMillis(date); assertEquals(date, ft.valueForDisplay(instant)); } @@ -205,7 +205,7 @@ public void testTermQuery() { ); MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; - long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + long instant = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date)).toInstant().toEpochMilli(); Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant, instant + 999), SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999) @@ -217,7 +217,7 @@ public void testTermQuery() { false, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), Resolution.MILLISECONDS, null, Collections.emptyMap() @@ -254,8 +254,8 @@ public void testRangeQuery() throws IOException { MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; - long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); - long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; + long instant1 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date2)).toInstant().toEpochMilli() + 999; Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant1, instant2), SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) @@ -280,7 +280,7 @@ public void testRangeQuery() throws IOException { false, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), Resolution.MILLISECONDS, null, Collections.emptyMap() @@ -326,8 +326,8 @@ public void testRangeQueryWithIndexSort() { MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; - long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); - long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; + long instant1 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date2)).toInstant().toEpochMilli() + 999; Query pointQuery = LongPoint.newRangeQuery("field", instant1, instant2); Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java index 4ebb160c07c8e..331bfb7b2ddf4 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java @@ -361,7 +361,12 @@ public void testSerializeDefaults() throws Exception { // if type is date_range we check that the mapper contains the default format and locale // otherwise it should not contain a locale or format - assertTrue(got, got.contains("\"format\":\"strict_date_optional_time||epoch_millis\"") == type.equals("date_range")); + assertTrue( + got, + got.contains("\"format\":\"strict_date_time_no_millis||strict_date_optional_time||epoch_millis\"") == type.equals( + "date_range" + ) + ); assertTrue(got, got.contains("\"locale\":" + "\"" + Locale.ROOT + "\"") == type.equals("date_range")); } } diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java index 668666a53cd7c..755d77c6ae392 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java @@ -265,7 +265,9 @@ public void testDateRangeQueryUsingMappingFormat() { ); assertThat( ex.getMessage(), - containsString("failed to parse date field [2016-15-06T15:29:50+08:00] with format [strict_date_optional_time||epoch_millis]") + containsString( + "failed to parse date field [2016-15-06T15:29:50+08:00] with format [strict_date_time_no_millis||strict_date_optional_time||epoch_millis]" + ) ); // setting mapping format which is compatible with those dates diff --git a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java index 50f785bdf4a34..e72be29b85b63 100644 --- a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java @@ -87,8 +87,8 @@ protected RangeQueryBuilder doCreateTestQueryBuilder() { ZonedDateTime start = now.minusMillis(randomIntBetween(0, 1000000)).atZone(ZoneOffset.UTC); ZonedDateTime end = now.plusMillis(randomIntBetween(0, 1000000)).atZone(ZoneOffset.UTC); query = new RangeQueryBuilder(randomFrom(DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, DATE_ALIAS_FIELD_NAME)); - query.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(start)); - query.to(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(end)); + query.from(DateFieldMapper.getDefaultDateTimeFormatter().format(start)); + query.to(DateFieldMapper.getDefaultDateTimeFormatter().format(end)); // Create timestamp option only then we have a date mapper, // otherwise we could trigger exception. if (createShardContext().getMapperService().fieldType(DATE_FIELD_NAME) != null) { diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java index 728b7162bf478..ce96623ea06df 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java @@ -135,7 +135,7 @@ private ValuesSourceConfig getVSConfig( indexed, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), resolution, null, Collections.emptyMap() @@ -184,7 +184,7 @@ public void testShortcutIsApplicable() throws IOException { assertNull(pointReaderShim(mockSearchContext(null), null, getVSConfig("number", resolution, false, context))); } // Check that we decode a dates "just like" the doc values instance. - Instant expected = Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2020-01-01T00:00:00Z")); + Instant expected = Instant.from(DateFieldMapper.getDefaultDateTimeFormatter().parse("2020-01-01T00:00:00Z")); byte[] scratch = new byte[8]; LongPoint.encodeDimension(DateFieldMapper.Resolution.MILLISECONDS.convert(expected), scratch, 0); assertThat( diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java index f5f4f3d4b0723..fdbc0160e51a3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java @@ -125,7 +125,7 @@ protected final DateFieldMapper.DateFieldType aggregableDateFieldType(boolean us isSearchable, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), useNanosecondResolution ? DateFieldMapper.Resolution.NANOSECONDS : DateFieldMapper.Resolution.MILLISECONDS, null, Collections.emptyMap() diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 4e8dc20c465a9..bca6623e66104 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -1245,7 +1245,7 @@ private void testSearchCase( } private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli(); } private static long asLong(String dateTime, DateFieldMapper.DateFieldType fieldType) { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java index 08adc6e3d550a..5c12d070824f2 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java @@ -1086,7 +1086,7 @@ private void testCase( } private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli(); } private static ZonedDateTime asZDT(String dateTime) { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java index 26ccc1075220b..96c8be1a25cc3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java @@ -273,7 +273,7 @@ private void testCase( true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), resolution, null, Collections.emptyMap() diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java index 761615ad1d7eb..dd7ae915c3b45 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -136,7 +136,7 @@ public void testDateFieldNanosecondResolution() throws IOException { true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), DateFieldMapper.Resolution.NANOSECONDS, null, Collections.emptyMap() @@ -167,7 +167,7 @@ public void testMissingDateWithDateField() throws IOException { true, false, true, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + DateFieldMapper.getDefaultDateTimeFormatter(), DateFieldMapper.Resolution.NANOSECONDS, null, Collections.emptyMap() diff --git a/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java index 3073cb6d35ddf..4b8d0ef83dd1b 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java @@ -144,6 +144,6 @@ public void testSameAggNames() throws IOException { } private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java index 17846234a09d1..5d7636208bd70 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java @@ -344,6 +344,6 @@ private void executeTestCase( } private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/pipeline/MovFnAggrgatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/pipeline/MovFnAggrgatorTests.java index 3ae00efaa6da3..d6abe50ea5201 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/pipeline/MovFnAggrgatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/pipeline/MovFnAggrgatorTests.java @@ -169,6 +169,6 @@ private void executeTestCase(Query query, DateHistogramAggregationBuilder aggBui } private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java index 1b348dc7d41a7..6b5ec838f401d 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java @@ -305,6 +305,6 @@ protected static Map createAfterKey(Object... fields) { } protected static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli(); } } From aac0f0916c0bf520fa80b1dce5ad527c20aad487 Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Thu, 5 Oct 2023 05:58:04 +0530 Subject: [PATCH 106/388] [Tracing Instrumentation] Add instrumentation in InboundHandler (#10143) * Add instrumentation in InboundHandler Signed-off-by: Gagan Juneja * Add CHANGELOG Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Empty-Commit Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja * Address review comment Signed-off-by: Gagan Juneja --------- Signed-off-by: Gagan Juneja Signed-off-by: Gagan Juneja Co-authored-by: Gagan Juneja --- CHANGELOG.md | 1 + .../transport/Netty4ModulePlugin.java | 6 +- .../transport/netty4/Netty4Transport.java | 6 +- .../Netty4SizeHeaderFrameDecoderTests.java | 4 +- .../netty4/NettyTransportMultiPortTests.java | 4 +- .../netty4/SimpleNetty4TransportTests.java | 4 +- .../discovery/ec2/Ec2DiscoveryTests.java | 3 +- .../discovery/ec2/Ec2RetriesTests.java | 3 +- .../telemetry/tracing/OTelSpan.java | 4 +- .../transport/nio/NioTransport.java | 6 +- .../transport/nio/NioTransportPlugin.java | 6 +- .../nio/SimpleNioTransportTests.java | 4 +- .../common/network/NetworkModule.java | 3 +- .../org/opensearch/plugins/NetworkPlugin.java | 3 +- .../telemetry/tracing/AttributeNames.java | 5 + .../telemetry/tracing/SpanBuilder.java | 23 +++ .../TraceableTcpTransportChannel.java | 112 ++++++++++++ .../transport/BaseTcpTransportChannel.java | 33 ++++ .../opensearch/transport/InboundHandler.java | 161 ++++++++++-------- .../transport/RequestHandlerRegistry.java | 4 +- .../opensearch/transport/TcpTransport.java | 7 +- .../transport/TcpTransportChannel.java | 14 +- .../node/tasks/TaskManagerTestCase.java | 3 +- ...TransportResyncReplicationActionTests.java | 3 +- .../BroadcastReplicationTests.java | 3 +- .../TransportReplicationActionTests.java | 3 +- .../common/network/NetworkModuleTests.java | 9 +- .../FileBasedSeedHostsProviderTests.java | 3 +- .../discovery/SeedHostsResolverTests.java | 15 +- .../extensions/ExtensionsManagerTests.java | 3 +- ...ExtensionTransportActionsHandlerTests.java | 3 +- .../RestInitializeExtensionActionTests.java | 3 +- .../rest/RestSendToExtensionActionTests.java | 3 +- .../transport/InboundHandlerTests.java | 4 +- .../transport/TcpTransportTests.java | 4 +- .../TransportServiceHandshakeTests.java | 3 +- .../test/transport/MockTransportService.java | 4 +- .../transport/nio/MockNioTransport.java | 6 +- .../transport/nio/MockNioTransportPlugin.java | 7 +- .../nio/SimpleMockNioTransportTests.java | 4 +- .../test/telemetry/tracing/MockSpan.java | 4 +- 41 files changed, 374 insertions(+), 129 deletions(-) create mode 100644 server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java create mode 100644 server/src/main/java/org/opensearch/transport/BaseTcpTransportChannel.java diff --git a/CHANGELOG.md b/CHANGELOG.md index dea5989d6d718..2f58e85b750d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122)) - Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246)) - Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200)) +- Add instrumentation in Inbound Handler. ([#100143](https://github.com/opensearch-project/OpenSearch/pull/10143)) - Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356)) ### Deprecated diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java index ca51d70702a82..1a34dfd2c9ee4 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java @@ -96,7 +96,8 @@ public Map> getTransports( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { return Collections.singletonMap( NETTY_TRANSPORT_NAME, @@ -108,7 +109,8 @@ public Map> getTransports( pageCacheRecycler, namedWriteableRegistry, circuitBreakerService, - getSharedGroupFactory(settings) + getSharedGroupFactory(settings), + tracer ) ); } diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java index f77c29c8bfa60..e76a227630dc1 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java @@ -50,6 +50,7 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Netty4NioSocketChannel; import org.opensearch.transport.NettyAllocator; @@ -131,9 +132,10 @@ public Netty4Transport( PageCacheRecycler pageCacheRecycler, NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService, - SharedGroupFactory sharedGroupFactory + SharedGroupFactory sharedGroupFactory, + Tracer tracer ) { - super(settings, version, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService); + super(settings, version, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService, tracer); Netty4Utils.setAvailableProcessors(OpenSearchExecutors.NODE_PROCESSORS_SETTING.get(settings)); NettyAllocator.logAllocatorDescriptionIfNeeded(); this.sharedGroupFactory = sharedGroupFactory; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java index 3e5f71f1464a1..c92ccba82835f 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -40,6 +40,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.SharedGroupFactory; @@ -86,7 +87,8 @@ public void startThreadPool() { recycler, new NamedWriteableRegistry(Collections.emptyList()), new NoneCircuitBreakerService(), - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ); nettyTransport.start(); diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java index 98a001b8ae4bb..7cca00db68559 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java @@ -40,6 +40,7 @@ import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -141,7 +142,8 @@ private TcpTransport startTransport(Settings settings, ThreadPool threadPool) { recycler, new NamedWriteableRegistry(Collections.emptyList()), new NoneCircuitBreakerService(), - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ); transport.start(); diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java index 35b19002dce8d..710b3ff6bd0ca 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.transport.MockTransportService; import org.opensearch.test.transport.StubbableTransport; import org.opensearch.transport.AbstractSimpleTransportTestCase; @@ -82,7 +83,8 @@ protected Transport build(Settings settings, final Version version, ClusterSetti PageCacheRecycler.NON_RECYCLING_INSTANCE, namedWriteableRegistry, new NoneCircuitBreakerService(), - new SharedGroupFactory(settings) + new SharedGroupFactory(settings), + NoopTracer.INSTANCE ) { @Override diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java index 6bed6564cfd36..02e1ff40f7ed6 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java @@ -92,7 +92,8 @@ protected MockTransportService createTransportService() { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, writableRegistry(), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override public TransportAddress[] addressesFromString(String address) { diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java index 3311ddc8842f2..ce097667f9c4b 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java @@ -77,7 +77,8 @@ protected MockTransportService createTransportService() { networkService, PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java index 8ad03d807d9da..fc917968579e1 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelSpan.java @@ -57,7 +57,9 @@ public void addAttribute(String key, Boolean value) { @Override public void setError(Exception exception) { - delegateSpan.setStatus(StatusCode.ERROR, exception.getMessage()); + if (exception != null) { + delegateSpan.setStatus(StatusCode.ERROR, exception.getMessage()); + } } @Override diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java index dfa72d6d59a0d..55920bab4efd3 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java @@ -52,6 +52,7 @@ import org.opensearch.nio.NioSelector; import org.opensearch.nio.NioSocketChannel; import org.opensearch.nio.ServerChannelContext; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TcpTransport; import org.opensearch.transport.TransportSettings; @@ -84,9 +85,10 @@ protected NioTransport( PageCacheRecycler pageCacheRecycler, NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService, - NioGroupFactory groupFactory + NioGroupFactory groupFactory, + Tracer tracer ) { - super(settings, version, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService); + super(settings, version, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService, tracer); this.pageAllocator = new PageAllocator(pageCacheRecycler); this.groupFactory = groupFactory; } diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java index ec266d76eff3d..d4be876867651 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java @@ -91,7 +91,8 @@ public Map> getTransports( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { return Collections.singletonMap( NIO_TRANSPORT_NAME, @@ -103,7 +104,8 @@ public Map> getTransports( pageCacheRecycler, namedWriteableRegistry, circuitBreakerService, - getNioGroupFactory(settings) + getNioGroupFactory(settings), + tracer ) ); } diff --git a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java index 24cc38c17a9d1..f5d1c618f5ace 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.transport.MockTransportService; import org.opensearch.test.transport.StubbableTransport; import org.opensearch.transport.AbstractSimpleTransportTestCase; @@ -81,7 +82,8 @@ protected Transport build(Settings settings, final Version version, ClusterSetti new MockPageCacheRecycler(settings), namedWriteableRegistry, new NoneCircuitBreakerService(), - new NioGroupFactory(settings, logger) + new NioGroupFactory(settings, logger), + NoopTracer.INSTANCE ) { @Override diff --git a/server/src/main/java/org/opensearch/common/network/NetworkModule.java b/server/src/main/java/org/opensearch/common/network/NetworkModule.java index 8870e26c373e9..0734659d8ee72 100644 --- a/server/src/main/java/org/opensearch/common/network/NetworkModule.java +++ b/server/src/main/java/org/opensearch/common/network/NetworkModule.java @@ -174,7 +174,8 @@ public NetworkModule( pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, - networkService + networkService, + tracer ); for (Map.Entry> entry : transportFactory.entrySet()) { registerTransport(entry.getKey(), entry.getValue()); diff --git a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java index f2f8e84f04e02..07df40bafe6a1 100644 --- a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java @@ -83,7 +83,8 @@ default Map> getTransports( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { return Collections.emptyMap(); } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java b/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java index e86b21ae0fd3b..a9514c298ef88 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java @@ -60,6 +60,11 @@ private AttributeNames() { */ public static final String TRANSPORT_TARGET_HOST = "target_host"; + /** + * Transport Service send request local host. + */ + public static final String TRANSPORT_HOST = "host"; + /** * Action Name. */ diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java index a93ce11f374fe..d97fbd371ab2a 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java @@ -13,6 +13,7 @@ import org.opensearch.http.HttpRequest; import org.opensearch.rest.RestRequest; import org.opensearch.telemetry.tracing.attributes.Attributes; +import org.opensearch.transport.TcpChannel; import org.opensearch.transport.Transport; import java.util.Arrays; @@ -127,4 +128,26 @@ private static Attributes buildSpanAttributes(String action, Transport.Connectio return attributes; } + /** + * Creates {@link SpanCreationContext} from Inbound Handler. + * @param action action. + * @param tcpChannel tcp channel. + * @return context + */ + public static SpanCreationContext from(String action, TcpChannel tcpChannel) { + return SpanCreationContext.server().name(createSpanName(action, tcpChannel)).attributes(buildSpanAttributes(action, tcpChannel)); + } + + private static String createSpanName(String action, TcpChannel tcpChannel) { + return action + SEPARATOR + (tcpChannel.getRemoteAddress() != null + ? tcpChannel.getRemoteAddress().getHostString() + : tcpChannel.getLocalAddress().getHostString()); + } + + private static Attributes buildSpanAttributes(String action, TcpChannel tcpChannel) { + Attributes attributes = Attributes.create().addAttribute(AttributeNames.TRANSPORT_ACTION, action); + attributes.addAttribute(AttributeNames.TRANSPORT_HOST, tcpChannel.getLocalAddress().getHostString()); + return attributes; + } + } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java new file mode 100644 index 0000000000000..333e06eb037cb --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.tracing.channels; + +import org.opensearch.Version; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.transport.BaseTcpTransportChannel; +import org.opensearch.transport.TcpTransportChannel; +import org.opensearch.transport.TransportChannel; + +import java.io.IOException; + +/** + * Tracer wrapped {@link TransportChannel} + */ +public class TraceableTcpTransportChannel extends BaseTcpTransportChannel { + + private final TransportChannel delegate; + private final Span span; + private final Tracer tracer; + + /** + * Constructor. + * @param delegate delegate + * @param span span + * @param tracer tracer + */ + public TraceableTcpTransportChannel(TcpTransportChannel delegate, Span span, Tracer tracer) { + super(delegate.getChannel()); + this.delegate = delegate; + this.span = span; + this.tracer = tracer; + } + + /** + * Factory method. + * + * @param delegate delegate + * @param span span + * @param tracer tracer + * @return transport channel + */ + public static TransportChannel create(TcpTransportChannel delegate, final Span span, final Tracer tracer) { + if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) { + delegate.getChannel().addCloseListener(new ActionListener() { + @Override + public void onResponse(Void unused) { + onFailure(null); + } + + @Override + public void onFailure(Exception e) { + span.addEvent("The TransportChannel was closed without sending the response"); + span.setError(e); + span.endSpan(); + } + }); + + return new TraceableTcpTransportChannel(delegate, span, tracer); + } else { + return delegate; + } + } + + @Override + public String getProfileName() { + return delegate.getProfileName(); + } + + @Override + public String getChannelType() { + return delegate.getChannelType(); + } + + @Override + public void sendResponse(TransportResponse response) throws IOException { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.sendResponse(response); + } catch (final IOException ex) { + span.setError(ex); + throw ex; + } finally { + span.endSpan(); + } + } + + @Override + public void sendResponse(Exception exception) throws IOException { + try (SpanScope scope = tracer.withSpanInScope(span)) { + delegate.sendResponse(exception); + } finally { + span.setError(exception); + span.endSpan(); + } + } + + @Override + public Version getVersion() { + return delegate.getVersion(); + } +} diff --git a/server/src/main/java/org/opensearch/transport/BaseTcpTransportChannel.java b/server/src/main/java/org/opensearch/transport/BaseTcpTransportChannel.java new file mode 100644 index 0000000000000..14e065d3350c7 --- /dev/null +++ b/server/src/main/java/org/opensearch/transport/BaseTcpTransportChannel.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.transport; + +/** + * Base class TcpTransportChannel + */ +public abstract class BaseTcpTransportChannel implements TransportChannel { + private final TcpChannel channel; + + /** + * Constructor. + * @param channel tcp channel + */ + public BaseTcpTransportChannel(TcpChannel channel) { + this.channel = channel; + } + + /** + * Returns {@link TcpChannel} + * @return TcpChannel + */ + public TcpChannel getChannel() { + return channel; + } + +} diff --git a/server/src/main/java/org/opensearch/transport/InboundHandler.java b/server/src/main/java/org/opensearch/transport/InboundHandler.java index 9f9232c18079a..c14a53e799319 100644 --- a/server/src/main/java/org/opensearch/transport/InboundHandler.java +++ b/server/src/main/java/org/opensearch/transport/InboundHandler.java @@ -46,6 +46,11 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanBuilder; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.telemetry.tracing.channels.TraceableTcpTransportChannel; import org.opensearch.threadpool.ThreadPool; import java.io.EOFException; @@ -74,6 +79,8 @@ public class InboundHandler { private volatile long slowLogThresholdMs = Long.MAX_VALUE; + private final Tracer tracer; + InboundHandler( ThreadPool threadPool, OutboundHandler outboundHandler, @@ -81,7 +88,8 @@ public class InboundHandler { TransportHandshaker handshaker, TransportKeepAlive keepAlive, Transport.RequestHandlers requestHandlers, - Transport.ResponseHandlers responseHandlers + Transport.ResponseHandlers responseHandlers, + Tracer tracer ) { this.threadPool = threadPool; this.outboundHandler = outboundHandler; @@ -90,6 +98,7 @@ public class InboundHandler { this.keepAlive = keepAlive; this.requestHandlers = requestHandlers; this.responseHandlers = responseHandlers; + this.tracer = tracer; } void setMessageListener(TransportMessageListener listener) { @@ -108,7 +117,6 @@ void inboundMessage(TcpChannel channel, InboundMessage message) throws Exception final long startTime = threadPool.relativeTimeInMillis(); channel.getChannelStats().markAccessed(startTime); TransportLogger.logInboundMessage(channel, message); - if (message.isPing()) { keepAlive.receiveKeepAlive(channel); } else { @@ -123,7 +131,6 @@ private void messageReceived(TcpChannel channel, InboundMessage message, long st final InetSocketAddress remoteAddress = channel.getRemoteAddress(); final Header header = message.getHeader(); assert header.needsToReadVariableHeader() == false; - ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext existing = threadContext.stashContext()) { // Place the context with the headers from the message @@ -165,6 +172,7 @@ private void messageReceived(TcpChannel channel, InboundMessage message, long st handleResponse(requestId, remoteAddress, EMPTY_STREAM_INPUT, handler); } } + } } finally { final long took = threadPool.relativeTimeInMillis() - startTime; @@ -184,80 +192,89 @@ private void handleRequest(TcpChannel channel, Head final String action = header.getActionName(); final long requestId = header.getRequestId(); final Version version = header.getVersion(); - if (header.isHandshake()) { - messageListener.onRequestReceived(requestId, action); - // Cannot short circuit handshakes - assert message.isShortCircuit() == false; - final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput()); - assertRemoteVersion(stream, header.getVersion()); - final TransportChannel transportChannel = new TcpTransportChannel( - outboundHandler, - channel, - action, - requestId, - version, - header.getFeatures(), - header.isCompressed(), - header.isHandshake(), - message.takeBreakerReleaseControl() - ); - try { - handshaker.handleHandshake(transportChannel, requestId, stream); - } catch (Exception e) { - if (Version.CURRENT.isCompatible(header.getVersion())) { - sendErrorResponse(action, transportChannel, e); - } else { - logger.warn( - new ParameterizedMessage( - "could not send error response to handshake received on [{}] using wire format version [{}], closing channel", - channel, - header.getVersion() - ), - e - ); - channel.close(); - } - } - } else { - final TransportChannel transportChannel = new TcpTransportChannel( - outboundHandler, - channel, - action, - requestId, - version, - header.getFeatures(), - header.isCompressed(), - header.isHandshake(), - message.takeBreakerReleaseControl() - ); - try { + Span span = tracer.startSpan(SpanBuilder.from(action, channel)); + try (SpanScope spanScope = tracer.withSpanInScope(span)) { + if (header.isHandshake()) { messageListener.onRequestReceived(requestId, action); - if (message.isShortCircuit()) { - sendErrorResponse(action, transportChannel, message.getException()); - } else { - final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput()); - assertRemoteVersion(stream, header.getVersion()); - final RequestHandlerRegistry reg = requestHandlers.getHandler(action); - assert reg != null; - - final T request = newRequest(requestId, action, stream, reg); - request.remoteAddress(new TransportAddress(channel.getRemoteAddress())); - checkStreamIsFullyConsumed(requestId, action, stream); - - final String executor = reg.getExecutor(); - if (ThreadPool.Names.SAME.equals(executor)) { - try { - reg.processMessageReceived(request, transportChannel); - } catch (Exception e) { - sendErrorResponse(reg.getAction(), transportChannel, e); - } + // Cannot short circuit handshakes + assert message.isShortCircuit() == false; + final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput()); + assertRemoteVersion(stream, header.getVersion()); + final TcpTransportChannel transportChannel = new TcpTransportChannel( + outboundHandler, + channel, + action, + requestId, + version, + header.getFeatures(), + header.isCompressed(), + header.isHandshake(), + message.takeBreakerReleaseControl() + ); + TransportChannel traceableTransportChannel = TraceableTcpTransportChannel.create(transportChannel, span, tracer); + try { + handshaker.handleHandshake(traceableTransportChannel, requestId, stream); + } catch (Exception e) { + if (Version.CURRENT.isCompatible(header.getVersion())) { + sendErrorResponse(action, traceableTransportChannel, e); } else { - threadPool.executor(executor).execute(new RequestHandler<>(reg, request, transportChannel)); + logger.warn( + new ParameterizedMessage( + "could not send error response to handshake received on [{}] using wire format version [{}], closing channel", + channel, + header.getVersion() + ), + e + ); + channel.close(); } } - } catch (Exception e) { - sendErrorResponse(action, transportChannel, e); + } else { + final TcpTransportChannel transportChannel = new TcpTransportChannel( + outboundHandler, + channel, + action, + requestId, + version, + header.getFeatures(), + header.isCompressed(), + header.isHandshake(), + message.takeBreakerReleaseControl() + ); + TransportChannel traceableTransportChannel = TraceableTcpTransportChannel.create(transportChannel, span, tracer); + try { + messageListener.onRequestReceived(requestId, action); + if (message.isShortCircuit()) { + sendErrorResponse(action, traceableTransportChannel, message.getException()); + } else { + final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput()); + assertRemoteVersion(stream, header.getVersion()); + final RequestHandlerRegistry reg = requestHandlers.getHandler(action); + assert reg != null; + + final T request = newRequest(requestId, action, stream, reg); + request.remoteAddress(new TransportAddress(channel.getRemoteAddress())); + checkStreamIsFullyConsumed(requestId, action, stream); + + final String executor = reg.getExecutor(); + if (ThreadPool.Names.SAME.equals(executor)) { + try { + reg.processMessageReceived(request, traceableTransportChannel); + } catch (Exception e) { + sendErrorResponse(reg.getAction(), traceableTransportChannel, e); + } + } else { + threadPool.executor(executor).execute(new RequestHandler<>(reg, request, traceableTransportChannel)); + } + } + } catch (Exception e) { + sendErrorResponse(action, traceableTransportChannel, e); + } } + } catch (Exception e) { + span.setError(e); + span.endSpan(); + throw e; } } diff --git a/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java b/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java index 464282730d2b2..98c182c562928 100644 --- a/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java +++ b/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java @@ -91,14 +91,14 @@ public void processMessageReceived(Request request, TransportChannel channel) th Releasable unregisterTask = () -> taskManager.unregister(task); try { - if (channel instanceof TcpTransportChannel && task instanceof CancellableTask) { + if (channel instanceof BaseTcpTransportChannel && task instanceof CancellableTask) { if (request instanceof ShardSearchRequest) { // on receiving request, update the inbound network time to reflect time spent in transit over the network ((ShardSearchRequest) request).setInboundNetworkTime( Math.max(0, System.currentTimeMillis() - ((ShardSearchRequest) request).getInboundNetworkTime()) ); } - final TcpChannel tcpChannel = ((TcpTransportChannel) channel).getChannel(); + final TcpChannel tcpChannel = ((BaseTcpTransportChannel) channel).getChannel(); final Releasable stopTracking = taskManager.startTrackingCancellableChannelTask(tcpChannel, (CancellableTask) task); unregisterTask = Releasables.wrap(unregisterTask, stopTracking); } diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index 7da7dcad13120..d0e6516973382 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -68,6 +68,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.node.Node; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -159,7 +160,8 @@ public TcpTransport( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { this.settings = settings; this.profileSettings = getProfileSettings(settings); @@ -208,7 +210,8 @@ public TcpTransport( handshaker, keepAlive, requestHandlers, - responseHandlers + responseHandlers, + tracer ); } diff --git a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java index 00702d08902a9..a7bedcf93e129 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java @@ -46,11 +46,10 @@ * * @opensearch.internal */ -public final class TcpTransportChannel implements TransportChannel { +public final class TcpTransportChannel extends BaseTcpTransportChannel { private final AtomicBoolean released = new AtomicBoolean(); private final OutboundHandler outboundHandler; - private final TcpChannel channel; private final String action; private final long requestId; private final Version version; @@ -70,9 +69,9 @@ public final class TcpTransportChannel implements TransportChannel { boolean isHandshake, Releasable breakerRelease ) { + super(channel); this.version = version; this.features = features; - this.channel = channel; this.outboundHandler = outboundHandler; this.action = action; this.requestId = requestId; @@ -83,7 +82,7 @@ public final class TcpTransportChannel implements TransportChannel { @Override public String getProfileName() { - return channel.getProfile(); + return getChannel().getProfile(); } @Override @@ -93,7 +92,7 @@ public void sendResponse(TransportResponse response) throws IOException { // update outbound network time with current time before sending response over network ((QuerySearchResult) response).getShardSearchRequest().setOutboundNetworkTime(System.currentTimeMillis()); } - outboundHandler.sendResponse(version, features, channel, requestId, action, response, compressResponse, isHandshake); + outboundHandler.sendResponse(version, features, getChannel(), requestId, action, response, compressResponse, isHandshake); } finally { release(false); } @@ -102,7 +101,7 @@ public void sendResponse(TransportResponse response) throws IOException { @Override public void sendResponse(Exception exception) throws IOException { try { - outboundHandler.sendErrorResponse(version, features, channel, requestId, action, exception); + outboundHandler.sendErrorResponse(version, features, getChannel(), requestId, action, exception); } finally { release(true); } @@ -131,7 +130,4 @@ public Version getVersion() { return version; } - public TcpChannel getChannel() { - return channel; - } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index 07e149dd72164..a3fa0f9cb16e4 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -211,7 +211,8 @@ public TestNode(String name, ThreadPool threadPool, Settings settings) { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java index fbac465f946f4..3bd8930064563 100644 --- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java @@ -135,7 +135,8 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception { new NetworkService(emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) ) { diff --git a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java index 77c9c64ad6611..19a9918fa4561 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java @@ -116,7 +116,8 @@ public void setUp() throws Exception { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - circuitBreakerService + circuitBreakerService, + NoopTracer.INSTANCE ); clusterService = createClusterService(threadPool); transportService = new TransportService( diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java index 0bee99f4d5656..03150d6e30755 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java @@ -1318,7 +1318,8 @@ public void testRetryOnReplicaWithRealTransport() throws Exception { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, namedWriteableRegistry, - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ); transportService = new MockTransportService( Settings.EMPTY, diff --git a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java index d28a4a51999e6..0ca118fe422a5 100644 --- a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java +++ b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java @@ -118,7 +118,8 @@ public Map> getTransports( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { return Collections.singletonMap("custom", custom); } @@ -176,7 +177,8 @@ public Map> getTransports( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { return Collections.singletonMap("default_custom", customTransport); } @@ -220,7 +222,8 @@ public Map> getTransports( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { return Collections.singletonMap("default_custom", customTransport); } diff --git a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java index 688a532a61c4a..f4515361a89b8 100644 --- a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java +++ b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java @@ -100,7 +100,8 @@ private void createTransportSvc() { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override public BoundTransportAddress boundAddress() { diff --git a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java index dc0829adac101..421f6c6fe279b 100644 --- a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java +++ b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java @@ -185,7 +185,8 @@ public void testRemovingLocalAddresses() { networkService, PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override @@ -237,7 +238,8 @@ public void testUnknownHost() { networkService, PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override @@ -292,7 +294,8 @@ public void testResolveTimeout() { networkService, PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override @@ -368,7 +371,8 @@ public void testCancellationOnClose() throws InterruptedException { networkService, PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override @@ -432,7 +436,8 @@ public void testInvalidHosts() throws IllegalAccessException { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override public BoundTransportAddress boundAddress() { diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index c61afdd5c5261..3c25dbdff3342 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -117,7 +117,8 @@ public void setup() throws Exception { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ); transportService = new MockTransportService( settings, diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java index 1dede94c68208..c4d2f81f7cf79 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java @@ -68,7 +68,8 @@ public void setup() throws Exception { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ); transportService = new MockTransportService( settings, diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index e237214ab88f5..0dae0ae1b4e0b 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -68,7 +68,8 @@ public void setup() throws Exception { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ); transportService = new MockTransportService( settings, diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index fe738ff7d85e6..9da976de7d7f6 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -79,7 +79,8 @@ public void setup() throws Exception { new NetworkService(Collections.emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(Collections.emptyList()), - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ); transportService = new MockTransportService( settings, diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java index 9a261c5745bc2..e002297911788 100644 --- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java @@ -50,6 +50,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.tasks.TaskManager; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -117,7 +118,8 @@ public void sendMessage(BytesReference reference, ActionListener listener) handshaker, keepAlive, requestHandlers, - responseHandlers + responseHandlers, + NoopTracer.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java index 06545b77c6d76..7ab78cca7d615 100644 --- a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java +++ b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java @@ -47,6 +47,7 @@ import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; @@ -255,7 +256,8 @@ private void testDefaultSeedAddresses(final Settings settings, Matcher> getTransports( PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, - NetworkService networkService + NetworkService networkService, + Tracer tracer ) { return Collections.singletonMap( MOCK_NIO_TRANSPORT_NAME, @@ -68,7 +70,8 @@ public Map> getTransports( networkService, pageCacheRecycler, namedWriteableRegistry, - circuitBreakerService + circuitBreakerService, + tracer ) ); } diff --git a/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java b/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java index fb77161a02aef..ce401ad99fad7 100644 --- a/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.transport.AbstractSimpleTransportTestCase; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.ConnectionProfile; @@ -71,7 +72,8 @@ protected Transport build(Settings settings, final Version version, ClusterSetti networkService, new MockPageCacheRecycler(settings), namedWriteableRegistry, - new NoneCircuitBreakerService() + new NoneCircuitBreakerService(), + NoopTracer.INSTANCE ) { @Override diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java index 4b661dc0ad0fe..c5d179f6412a8 100644 --- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java +++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/MockSpan.java @@ -166,7 +166,9 @@ public Long getEndTime() { } public void setError(Exception exception) { - putMetadata("ERROR", exception.getMessage()); + if (exception != null) { + putMetadata("ERROR", exception.getMessage()); + } } private static class IdGenerator { From bf5177911ca42135cb5d84ebebd6db38bb112912 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 17:55:16 -0700 Subject: [PATCH 107/388] Bump org.apache.avro:avro from 1.11.2 to 1.11.3 in /plugins/repository-hdfs (#10210) * Bump org.apache.avro:avro in /plugins/repository-hdfs Bumps org.apache.avro:avro from 1.11.2 to 1.11.3. --- updated-dependencies: - dependency-name: org.apache.avro:avro dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 | 1 - plugins/repository-hdfs/licenses/avro-1.11.3.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/avro-1.11.3.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f58e85b750d1..1c99c2ad1b99f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302)) - Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303)) - Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301)) +- Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index e2f5707ca4393..8d9ff01f5db65 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -66,7 +66,7 @@ dependencies { } api 'org.apache.htrace:htrace-core4:4.2.0-incubating' api "org.apache.logging.log4j:log4j-core:${versions.log4j}" - api 'org.apache.avro:avro:1.11.2' + api 'org.apache.avro:avro:1.11.3' api 'com.google.code.gson:gson:2.10.1' runtimeOnly "com.google.guava:guava:${versions.guava}" api "commons-logging:commons-logging:${versions.commonslogging}" diff --git a/plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 b/plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 deleted file mode 100644 index ce1a894e0ce6d..0000000000000 --- a/plugins/repository-hdfs/licenses/avro-1.11.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -97e62e8be2b37e849f1bdb5a4f08121d47cc9806 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/avro-1.11.3.jar.sha1 b/plugins/repository-hdfs/licenses/avro-1.11.3.jar.sha1 new file mode 100644 index 0000000000000..fb43ecbcf22c9 --- /dev/null +++ b/plugins/repository-hdfs/licenses/avro-1.11.3.jar.sha1 @@ -0,0 +1 @@ +02b463409b373bff9ece09f54a43d42da5cea55a \ No newline at end of file From ca0dae61cb1845f2a2543d051a00be946635fc83 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 4 Oct 2023 18:19:02 -0700 Subject: [PATCH 108/388] Update multipart download path to first write to temp files (#10347) * Update multipart download path to write to temp files. This change updates ReadContextListener to first write parts to a temp location until all parts have been received. Signed-off-by: Marc Handalian * Suppress forbidden IOUtils.fsync Signed-off-by: Marc Handalian * Remove unnecessary logging format Signed-off-by: Marc Handalian * sync directory after file rename Signed-off-by: Marc Handalian * Remove flaky threadpool terminate test Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../read/listener/ReadContextListener.java | 49 +++++++++++++-- .../listener/ReadContextListenerTests.java | 63 +++++++++++++++++++ 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java index 2914fd0c440fa..c77f2384ace0d 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java +++ b/server/src/main/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListener.java @@ -10,7 +10,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.IOUtils; import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.core.action.ActionListener; @@ -20,6 +23,8 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Collection; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; @@ -33,9 +38,11 @@ @InternalApi public class ReadContextListener implements ActionListener { private static final Logger logger = LogManager.getLogger(ReadContextListener.class); - + private static final String DOWNLOAD_PREFIX = "download."; private final String blobName; private final Path fileLocation; + private final String tmpFileName; + private final Path tmpFileLocation; private final ActionListener completionListener; private final ThreadPool threadPool; private final UnaryOperator rateLimiter; @@ -55,6 +62,8 @@ public ReadContextListener( this.threadPool = threadPool; this.rateLimiter = rateLimiter; this.maxConcurrentStreams = maxConcurrentStreams; + this.tmpFileName = DOWNLOAD_PREFIX + UUIDs.randomBase64UUID() + "." + blobName; + this.tmpFileLocation = fileLocation.getParent().resolve(tmpFileName); } @Override @@ -62,15 +71,12 @@ public void onResponse(ReadContext readContext) { logger.debug("Received {} parts for blob {}", readContext.getNumberOfParts(), blobName); final int numParts = readContext.getNumberOfParts(); final AtomicBoolean anyPartStreamFailed = new AtomicBoolean(false); - final GroupedActionListener groupedListener = new GroupedActionListener<>( - ActionListener.wrap(r -> completionListener.onResponse(blobName), completionListener::onFailure), - numParts - ); + final GroupedActionListener groupedListener = new GroupedActionListener<>(getFileCompletionListener(), numParts); final Queue queue = new ConcurrentLinkedQueue<>(readContext.getPartStreams()); final StreamPartProcessor processor = new StreamPartProcessor( queue, anyPartStreamFailed, - fileLocation, + tmpFileLocation, groupedListener, threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY), rateLimiter @@ -80,6 +86,37 @@ public void onResponse(ReadContext readContext) { } } + @SuppressForbidden(reason = "need to fsync once all parts received") + private ActionListener> getFileCompletionListener() { + return ActionListener.wrap(response -> { + logger.trace("renaming temp file [{}] to [{}]", tmpFileLocation, fileLocation); + try { + IOUtils.fsync(tmpFileLocation, false); + Files.move(tmpFileLocation, fileLocation, StandardCopyOption.ATOMIC_MOVE); + // sync parent dir metadata + IOUtils.fsync(fileLocation.getParent(), true); + completionListener.onResponse(blobName); + } catch (IOException e) { + logger.error("Unable to rename temp file + " + tmpFileLocation, e); + completionListener.onFailure(e); + } + }, e -> { + try { + Files.deleteIfExists(tmpFileLocation); + } catch (IOException ex) { + logger.warn("Unable to clean temp file {}", tmpFileLocation); + } + completionListener.onFailure(e); + }); + } + + /* + * For Tests + */ + Path getTmpFileLocation() { + return tmpFileLocation; + } + @Override public void onFailure(Exception e) { completionListener.onFailure(e); diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java index 7e4c96cbadcda..0163c2275e7f4 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java @@ -130,6 +130,7 @@ public int available() { countDownLatch.await(); assertFalse(Files.exists(fileLocation)); + assertFalse(Files.exists(readContextListener.getTmpFileLocation())); } public void testReadContextListenerException() { @@ -149,6 +150,68 @@ public void testReadContextListenerException() { assertEquals(exception, listener.getException()); } + public void testWriteToTempFile() throws Exception { + final String fileName = UUID.randomUUID().toString(); + Path fileLocation = path.resolve(fileName); + List blobPartStreams = initializeBlobPartStreams(); + CountDownLatch countDownLatch = new CountDownLatch(1); + ActionListener completionListener = new LatchedActionListener<>(new PlainActionFuture<>(), countDownLatch); + ReadContextListener readContextListener = new ReadContextListener( + TEST_SEGMENT_FILE, + fileLocation, + completionListener, + threadPool, + UnaryOperator.identity(), + MAX_CONCURRENT_STREAMS + ); + ByteArrayInputStream assertingStream = new ByteArrayInputStream(randomByteArrayOfLength(PART_SIZE)) { + @Override + public int read(byte[] b) throws IOException { + assertTrue("parts written to temp file location", Files.exists(readContextListener.getTmpFileLocation())); + return super.read(b); + } + }; + blobPartStreams.add( + NUMBER_OF_PARTS, + () -> CompletableFuture.supplyAsync( + () -> new InputStreamContainer(assertingStream, PART_SIZE, PART_SIZE * NUMBER_OF_PARTS), + threadPool.generic() + ) + ); + ReadContext readContext = new ReadContext((long) (PART_SIZE + 1) * NUMBER_OF_PARTS + 1, blobPartStreams, null); + readContextListener.onResponse(readContext); + + countDownLatch.await(); + assertTrue(Files.exists(fileLocation)); + assertFalse(Files.exists(readContextListener.getTmpFileLocation())); + } + + public void testWriteToTempFile_alreadyExists_replacesFile() throws Exception { + final String fileName = UUID.randomUUID().toString(); + Path fileLocation = path.resolve(fileName); + // create an empty file at location. + Files.createFile(fileLocation); + assertEquals(0, Files.readAllBytes(fileLocation).length); + List blobPartStreams = initializeBlobPartStreams(); + CountDownLatch countDownLatch = new CountDownLatch(1); + ActionListener completionListener = new LatchedActionListener<>(new PlainActionFuture<>(), countDownLatch); + ReadContextListener readContextListener = new ReadContextListener( + TEST_SEGMENT_FILE, + fileLocation, + completionListener, + threadPool, + UnaryOperator.identity(), + MAX_CONCURRENT_STREAMS + ); + ReadContext readContext = new ReadContext((long) (PART_SIZE + 1) * NUMBER_OF_PARTS, blobPartStreams, null); + readContextListener.onResponse(readContext); + + countDownLatch.await(); + assertTrue(Files.exists(fileLocation)); + assertEquals(50, Files.readAllBytes(fileLocation).length); + assertFalse(Files.exists(readContextListener.getTmpFileLocation())); + } + private List initializeBlobPartStreams() { List blobPartStreams = new ArrayList<>(); for (int partNumber = 0; partNumber < NUMBER_OF_PARTS; partNumber++) { From 179c4ef94d0176de5161a0e620bca9609dd2a051 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 5 Oct 2023 06:52:10 +0530 Subject: [PATCH 109/388] [Remote Store] Add support to reload repository metadata inplace (#9569) --------- Signed-off-by: Bhumika Saini --- CHANGELOG.md | 1 + .../repositories/url/URLRepository.java | 2 +- .../repositories/azure/AzureRepository.java | 9 +- .../gcs/GoogleCloudStorageRepository.java | 9 +- .../repositories/hdfs/HdfsRepository.java | 2 +- .../repositories/s3/S3BlobStore.java | 28 +- .../repositories/s3/S3Repository.java | 245 ++++++++++++------ .../repositories/s3/S3RepositoryPlugin.java | 3 +- .../opensearch/repositories/s3/S3Service.java | 5 +- .../repositories/s3/S3RepositoryTests.java | 10 +- .../RemoteStoreBaseIntegTestCase.java | 1 - .../remotestore/RemoteStoreRestoreIT.java | 47 +++- .../common/blobstore/BlobStore.java | 7 + .../repositories/RepositoriesService.java | 34 ++- .../opensearch/repositories/Repository.java | 18 ++ .../blobstore/BlobStoreRepository.java | 54 ++-- .../blobstore/MeteredBlobStoreRepository.java | 11 +- .../repositories/fs/FsRepository.java | 46 ++-- .../fs/ReloadableFsRepository.java | 51 ++++ .../RepositoriesServiceTests.java | 18 +- .../blobstore/BlobStoreRepositoryTests.java | 7 +- .../fs/ReloadableFsRepositoryTests.java | 119 +++++++++ .../MockEventuallyConsistentRepository.java | 2 +- 23 files changed, 545 insertions(+), 184 deletions(-) create mode 100644 server/src/main/java/org/opensearch/repositories/fs/ReloadableFsRepository.java create mode 100644 server/src/test/java/org/opensearch/repositories/fs/ReloadableFsRepositoryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c99c2ad1b99f..aea4a45b60ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,6 +129,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200)) - Add instrumentation in Inbound Handler. ([#100143](https://github.com/opensearch-project/OpenSearch/pull/10143)) - Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356)) +- [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569)) ### Deprecated diff --git a/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java index 9e9d94c8e8fc0..4c8d8aab4532b 100644 --- a/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java +++ b/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java @@ -113,7 +113,7 @@ public URLRepository( ClusterService clusterService, RecoverySettings recoverySettings ) { - super(metadata, false, namedXContentRegistry, clusterService, recoverySettings); + super(metadata, namedXContentRegistry, clusterService, recoverySettings); if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) == false) { throw new RepositoryException(metadata.name(), "missing url"); diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java index 381a35bbc11e1..47a5536a6cd8a 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java @@ -116,14 +116,7 @@ public AzureRepository( final ClusterService clusterService, final RecoverySettings recoverySettings ) { - super( - metadata, - COMPRESS_SETTING.get(metadata.settings()), - namedXContentRegistry, - clusterService, - recoverySettings, - buildLocation(metadata) - ); + super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildLocation(metadata)); this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings()); this.storageService = storageService; diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java index 13671bc2aa8d6..f6d078868b875 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -94,14 +94,7 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { final ClusterService clusterService, final RecoverySettings recoverySettings ) { - super( - metadata, - getSetting(COMPRESS_SETTING, metadata), - namedXContentRegistry, - clusterService, - recoverySettings, - buildLocation(metadata) - ); + super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildLocation(metadata)); this.storageService = storageService; String basePath = BASE_PATH.get(metadata.settings()); diff --git a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java index b28d28d76cfde..f0ffec5713c1d 100644 --- a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java @@ -83,7 +83,7 @@ public HdfsRepository( final ClusterService clusterService, final RecoverySettings recoverySettings ) { - super(metadata, COMPRESS_SETTING.get(metadata.settings()), namedXContentRegistry, clusterService, recoverySettings); + super(metadata, namedXContentRegistry, clusterService, recoverySettings); this.environment = environment; this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java index 8a5b92d71bb45..3dd373b5b9f32 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java @@ -50,6 +50,12 @@ import java.util.Locale; import java.util.Map; +import static org.opensearch.repositories.s3.S3Repository.BUCKET_SETTING; +import static org.opensearch.repositories.s3.S3Repository.BUFFER_SIZE_SETTING; +import static org.opensearch.repositories.s3.S3Repository.CANNED_ACL_SETTING; +import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_SETTING; +import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING; + class S3BlobStore implements BlobStore { private static final Logger logger = LogManager.getLogger(S3BlobStore.class); @@ -58,17 +64,17 @@ class S3BlobStore implements BlobStore { private final S3AsyncService s3AsyncService; - private final String bucket; + private volatile String bucket; - private final ByteSizeValue bufferSize; + private volatile ByteSizeValue bufferSize; - private final boolean serverSideEncryption; + private volatile boolean serverSideEncryption; - private final ObjectCannedACL cannedACL; + private volatile ObjectCannedACL cannedACL; - private final StorageClass storageClass; + private volatile StorageClass storageClass; - private final RepositoryMetadata repositoryMetadata; + private volatile RepositoryMetadata repositoryMetadata; private final StatsMetricPublisher statsMetricPublisher = new StatsMetricPublisher(); @@ -105,8 +111,14 @@ class S3BlobStore implements BlobStore { this.priorityExecutorBuilder = priorityExecutorBuilder; } - public boolean isMultipartUploadEnabled() { - return multipartUploadEnabled; + @Override + public void reload(RepositoryMetadata repositoryMetadata) { + this.repositoryMetadata = repositoryMetadata; + this.bucket = BUCKET_SETTING.get(repositoryMetadata.settings()); + this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(repositoryMetadata.settings()); + this.bufferSize = BUFFER_SIZE_SETTING.get(repositoryMetadata.settings()); + this.cannedACL = initCannedACL(CANNED_ACL_SETTING.get(repositoryMetadata.settings())); + this.storageClass = initStorageClass(STORAGE_CLASS_SETTING.get(repositoryMetadata.settings())); } @Override diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java index 8e69fc39f128e..a69309d9e7a6f 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java @@ -32,6 +32,9 @@ package org.opensearch.repositories.s3; +import software.amazon.awssdk.services.s3.model.ObjectCannedACL; +import software.amazon.awssdk.services.s3.model.StorageClass; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; @@ -41,9 +44,11 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.SecureSetting; import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; import org.opensearch.core.common.settings.SecureString; @@ -62,9 +67,11 @@ import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.threadpool.Scheduler; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -79,7 +86,6 @@ *

{@code concurrent_streams}
Number of concurrent read/write stream (per repository on each node). Defaults to 5.
*
{@code chunk_size}
*
Large file can be divided into chunks. This parameter specifies the chunk size. Defaults to not chucked.
- *
{@code compress}
If set to true metadata files will be stored compressed. Defaults to false.
* */ class S3Repository extends MeteredBlobStoreRepository { @@ -205,31 +211,27 @@ class S3Repository extends MeteredBlobStoreRepository { private final S3Service service; - private final String bucket; - - private final ByteSizeValue bufferSize; - - private final ByteSizeValue chunkSize; + private volatile String bucket; - private final BlobPath basePath; + private volatile ByteSizeValue bufferSize; - private final boolean serverSideEncryption; + private volatile ByteSizeValue chunkSize; - private final String storageClass; + private volatile BlobPath basePath; - private final String cannedACL; + private volatile boolean serverSideEncryption; - private final RepositoryMetadata repositoryMetadata; + private volatile String storageClass; + private volatile String cannedACL; private final AsyncTransferManager asyncUploadUtils; private final S3AsyncService s3AsyncService; private final boolean multipartUploadEnabled; private final AsyncExecutorContainer priorityExecutorBuilder; private final AsyncExecutorContainer normalExecutorBuilder; + private final Path pluginConfigPath; - /** - * Constructs an s3 backed repository - */ + // Used by test classes S3Repository( final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, @@ -242,77 +244,48 @@ class S3Repository extends MeteredBlobStoreRepository { final S3AsyncService s3AsyncService, final boolean multipartUploadEnabled ) { - super( + this( metadata, - COMPRESS_SETTING.get(metadata.settings()), namedXContentRegistry, + service, clusterService, recoverySettings, - buildLocation(metadata) + asyncUploadUtils, + priorityExecutorBuilder, + normalExecutorBuilder, + s3AsyncService, + multipartUploadEnabled, + Path.of("") ); + } + + /** + * Constructs an s3 backed repository + */ + S3Repository( + final RepositoryMetadata metadata, + final NamedXContentRegistry namedXContentRegistry, + final S3Service service, + final ClusterService clusterService, + final RecoverySettings recoverySettings, + final AsyncTransferManager asyncUploadUtils, + final AsyncExecutorContainer priorityExecutorBuilder, + final AsyncExecutorContainer normalExecutorBuilder, + final S3AsyncService s3AsyncService, + final boolean multipartUploadEnabled, + Path pluginConfigPath + ) { + super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildLocation(metadata)); this.service = service; this.s3AsyncService = s3AsyncService; this.multipartUploadEnabled = multipartUploadEnabled; - - this.repositoryMetadata = metadata; + this.pluginConfigPath = pluginConfigPath; this.asyncUploadUtils = asyncUploadUtils; this.priorityExecutorBuilder = priorityExecutorBuilder; this.normalExecutorBuilder = normalExecutorBuilder; - // Parse and validate the user's S3 Storage Class setting - this.bucket = BUCKET_SETTING.get(metadata.settings()); - if (bucket == null) { - throw new RepositoryException(metadata.name(), "No bucket defined for s3 repository"); - } - - this.bufferSize = BUFFER_SIZE_SETTING.get(metadata.settings()); - this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); - - // We make sure that chunkSize is bigger or equal than/to bufferSize - if (this.chunkSize.getBytes() < bufferSize.getBytes()) { - throw new RepositoryException( - metadata.name(), - CHUNK_SIZE_SETTING.getKey() - + " (" - + this.chunkSize - + ") can't be lower than " - + BUFFER_SIZE_SETTING.getKey() - + " (" - + bufferSize - + ")." - ); - } - - final String basePath = BASE_PATH_SETTING.get(metadata.settings()); - if (Strings.hasLength(basePath)) { - this.basePath = new BlobPath().add(basePath); - } else { - this.basePath = BlobPath.cleanPath(); - } - - this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings()); - - this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings()); - this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings()); - - if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) { - // provided repository settings - deprecationLogger.deprecate( - "s3_repository_secret_settings", - "Using s3 access/secret key from repository settings. Instead " - + "store these in named clients and the opensearch keystore for secure settings." - ); - } - - logger.debug( - "using bucket [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], cannedACL [{}], storageClass [{}]", - bucket, - chunkSize, - serverSideEncryption, - bufferSize, - cannedACL, - storageClass - ); + validateRepositoryMetadata(metadata); + readRepositoryMetadata(); } private static Map buildLocation(RepositoryMetadata metadata) { @@ -367,14 +340,14 @@ protected S3BlobStore createBlobStore() { bufferSize, cannedACL, storageClass, - repositoryMetadata, + metadata, asyncUploadUtils, priorityExecutorBuilder, normalExecutorBuilder ); } - // only use for testing + // only use for testing (S3RepositoryTests) @Override protected BlobStore getBlobStore() { return super.getBlobStore(); @@ -385,6 +358,128 @@ public BlobPath basePath() { return basePath; } + @Override + public boolean isReloadable() { + return true; + } + + @Override + public void reload(RepositoryMetadata newRepositoryMetadata) { + if (isReloadable() == false) { + return; + } + + // Reload configs for S3Repository + super.reload(newRepositoryMetadata); + readRepositoryMetadata(); + + // Reload configs for S3RepositoryPlugin + final Map clientsSettings = S3ClientSettings.load(metadata.settings(), pluginConfigPath); + service.refreshAndClearCache(clientsSettings); + s3AsyncService.refreshAndClearCache(clientsSettings); + + // Reload configs for S3BlobStore + BlobStore blobStore = getBlobStore(); + blobStore.reload(metadata); + } + + /** + * Reloads the values derived from the Repository Metadata + */ + private void readRepositoryMetadata() { + this.bucket = BUCKET_SETTING.get(metadata.settings()); + this.bufferSize = BUFFER_SIZE_SETTING.get(metadata.settings()); + this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); + final String basePath = BASE_PATH_SETTING.get(metadata.settings()); + if (Strings.hasLength(basePath)) { + this.basePath = new BlobPath().add(basePath); + } else { + this.basePath = BlobPath.cleanPath(); + } + + this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings()); + this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings()); + this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings()); + if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) { + // provided repository settings + deprecationLogger.deprecate( + "s3_repository_secret_settings", + "Using s3 access/secret key from repository settings. Instead " + + "store these in named clients and the opensearch keystore for secure settings." + ); + } + + logger.debug( + "using bucket [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], cannedACL [{}], storageClass [{}]", + bucket, + chunkSize, + serverSideEncryption, + bufferSize, + cannedACL, + storageClass + ); + } + + @Override + public void validateMetadata(RepositoryMetadata newRepositoryMetadata) { + super.validateMetadata(newRepositoryMetadata); + validateRepositoryMetadata(newRepositoryMetadata); + } + + private void validateRepositoryMetadata(RepositoryMetadata newRepositoryMetadata) { + Settings settings = newRepositoryMetadata.settings(); + if (BUCKET_SETTING.get(settings) == null) { + throw new RepositoryException(newRepositoryMetadata.name(), "No bucket defined for s3 repository"); + } + + // We make sure that chunkSize is bigger or equal than/to bufferSize + if (CHUNK_SIZE_SETTING.get(settings).getBytes() < BUFFER_SIZE_SETTING.get(settings).getBytes()) { + throw new RepositoryException( + newRepositoryMetadata.name(), + CHUNK_SIZE_SETTING.getKey() + + " (" + + CHUNK_SIZE_SETTING.get(settings) + + ") can't be lower than " + + BUFFER_SIZE_SETTING.getKey() + + " (" + + BUFFER_SIZE_SETTING.get(settings) + + ")." + ); + } + + validateStorageClass(STORAGE_CLASS_SETTING.get(settings)); + validateCannedACL(CANNED_ACL_SETTING.get(settings)); + } + + private static void validateStorageClass(String storageClassStringValue) { + if ((storageClassStringValue == null) || storageClassStringValue.equals("")) { + return; + } + + final StorageClass storageClass = StorageClass.fromValue(storageClassStringValue.toUpperCase(Locale.ENGLISH)); + if (storageClass.equals(StorageClass.GLACIER)) { + throw new BlobStoreException("Glacier storage class is not supported"); + } + + if (storageClass == StorageClass.UNKNOWN_TO_SDK_VERSION) { + throw new BlobStoreException("`" + storageClassStringValue + "` is not a valid S3 Storage Class."); + } + } + + private static void validateCannedACL(String cannedACLStringValue) { + if ((cannedACLStringValue == null) || cannedACLStringValue.equals("")) { + return; + } + + for (final ObjectCannedACL cur : ObjectCannedACL.values()) { + if (cur.toString().equalsIgnoreCase(cannedACLStringValue)) { + return; + } + } + + throw new BlobStoreException("cannedACL is not valid: [" + cannedACLStringValue + "]"); + } + @Override protected ByteSizeValue chunkSize() { return chunkSize; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 6ef60474afe8c..a80ee0ca35fae 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -182,7 +182,8 @@ protected S3Repository createRepository( priorityExecutorBuilder, normalExecutorBuilder, s3AsyncService, - S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING.get(clusterService.getSettings()) + S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING.get(clusterService.getSettings()), + configPath ); } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index b13672b4179f8..b1b3e19eac275 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -90,6 +90,7 @@ import java.security.SecureRandom; import java.time.Duration; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static java.util.Collections.emptyMap; @@ -100,7 +101,7 @@ class S3Service implements Closeable { private static final String DEFAULT_S3_ENDPOINT = "s3.amazonaws.com"; - private volatile Map clientsCache = emptyMap(); + private volatile Map clientsCache = new ConcurrentHashMap<>(); /** * Client settings calculated from static configuration and settings in the keystore. @@ -111,7 +112,7 @@ class S3Service implements Closeable { * Client settings derived from those in {@link #staticClientSettings} by combining them with settings * in the {@link RepositoryMetadata}. */ - private volatile Map derivedClientSettings = emptyMap(); + private volatile Map derivedClientSettings = new ConcurrentHashMap<>(); S3Service(final Path configPath) { staticClientSettings = MapBuilder.newMapBuilder() diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java index 9ea8d98505161..e65ca69a5047b 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java @@ -125,7 +125,8 @@ public void testBasePathSetting() { } public void testDefaultBufferSize() { - final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", Settings.EMPTY); + Settings settings = Settings.builder().build(); + final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", settings); try (S3Repository s3repo = createS3Repo(metadata)) { assertThat(s3repo.getBlobStore(), is(nullValue())); s3repo.start(); @@ -136,6 +137,13 @@ public void testDefaultBufferSize() { } } + public void testIsReloadable() { + final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", Settings.EMPTY); + try (S3Repository s3repo = createS3Repo(metadata)) { + assertTrue(s3repo.isReloadable()); + } + } + public void testRestrictedSettingsDefault() { final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", Settings.EMPTY); try (S3Repository s3repo = createS3Repo(metadata)) { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index bf536557b1485..e2ef5f85abc74 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -271,7 +271,6 @@ public static Settings buildRemoteStoreNodeAttributes( if (withRateLimiterAttributes) { settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) - .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "4kb") .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 65335f444a2df..7626e3dba6424 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -12,17 +12,26 @@ import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThan; @@ -388,14 +397,41 @@ public void testRTSRestoreDataOnlyInTranslog() throws Exception { public void testRateLimitedRemoteDownloads() throws Exception { clusterSettingsSuppliedByTest = true; int shardCount = randomIntBetween(1, 3); + Path segmentRepoPath = randomRepoPath(); + Path tlogRepoPath = randomRepoPath(); prepareCluster( 1, 3, INDEX_NAME, 0, shardCount, - buildRemoteStoreNodeAttributes(REPOSITORY_NAME, randomRepoPath(), REPOSITORY_2_NAME, randomRepoPath(), true) + buildRemoteStoreNodeAttributes(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, tlogRepoPath, true) ); + + // validate inplace repository metadata update + ClusterService clusterService = internalCluster().getInstance(ClusterService.class); + DiscoveryNode node = clusterService.localNode(); + String settingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); + Map settingsMap = node.getAttributes() + .keySet() + .stream() + .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> node.getAttributes().get(key))); + Settings.Builder settings = Settings.builder(); + settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + settings.put("location", segmentRepoPath).put("max_remote_download_bytes_per_sec", 4, ByteSizeUnit.KB); + + assertAcked(client().admin().cluster().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(settings).get()); + + for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) { + Repository segmentRepo = repositoriesService.repository(REPOSITORY_NAME); + assertEquals("4096b", segmentRepo.getMetadata().settings().get("max_remote_download_bytes_per_sec")); + } + Map indexStats = indexData(5, false, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); @@ -414,6 +450,15 @@ public void testRateLimitedRemoteDownloads() throws Exception { assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); assertEquals(0, getNumShards(INDEX_NAME).numReplicas); verifyRestoredData(indexStats, INDEX_NAME); + + // revert repo metadata to pass asserts on repo metadata vs. node attrs during teardown + // https://github.com/opensearch-project/OpenSearch/pull/9569#discussion_r1345668700 + settings.remove("max_remote_download_bytes_per_sec"); + assertAcked(client().admin().cluster().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(settings).get()); + for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) { + Repository segmentRepo = repositoriesService.repository(REPOSITORY_NAME); + assertNull(segmentRepo.getMetadata().settings().get("max_remote_download_bytes_per_sec")); + } } // TODO: Restore flow - index aliases diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java index ab40b1e2a082e..2ee3e9557b354 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java +++ b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java @@ -31,6 +31,8 @@ package org.opensearch.common.blobstore; +import org.opensearch.cluster.metadata.RepositoryMetadata; + import java.io.Closeable; import java.util.Collections; import java.util.Map; @@ -53,4 +55,9 @@ public interface BlobStore extends Closeable { default Map stats() { return Collections.emptyMap(); } + + /** + * Reload the blob store inplace + */ + default void reload(RepositoryMetadata repositoryMetadata) {} } diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 0361a71116a16..72266c053a1ae 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -474,19 +474,29 @@ public void applyClusterState(ClusterChangedEvent event) { if (previousMetadata.type().equals(repositoryMetadata.type()) == false || previousMetadata.settings().equals(repositoryMetadata.settings()) == false) { // Previous version is different from the version in settings - logger.debug("updating repository [{}]", repositoryMetadata.name()); - closeRepository(repository); - archiveRepositoryStats(repository, state.version()); - repository = null; - try { - repository = createRepository(repositoryMetadata, typesRegistry); - } catch (RepositoryException ex) { - // TODO: this catch is bogus, it means the old repo is already closed, - // but we have nothing to replace it - logger.warn( - () -> new ParameterizedMessage("failed to change repository [{}]", repositoryMetadata.name()), - ex + if (repository.isSystemRepository() && repository.isReloadable()) { + logger.debug( + "updating repository [{}] in-place to use new metadata [{}]", + repositoryMetadata.name(), + repositoryMetadata ); + repository.validateMetadata(repositoryMetadata); + repository.reload(repositoryMetadata); + } else { + logger.debug("updating repository [{}]", repositoryMetadata.name()); + closeRepository(repository); + archiveRepositoryStats(repository, state.version()); + repository = null; + try { + repository = createRepository(repositoryMetadata, typesRegistry); + } catch (RepositoryException ex) { + // TODO: this catch is bogus, it means the old repo is already closed, + // but we have nothing to replace it + logger.warn( + () -> new ParameterizedMessage("failed to change repository [{}]", repositoryMetadata.name()), + ex + ); + } } } } else { diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 02ba60cb23c4d..74cda9bcc3393 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -451,4 +451,22 @@ default void cloneRemoteStoreIndexShardSnapshot( default Map adaptUserMetadata(Map userMetadata) { return userMetadata; } + + /** + * Checks if the repository can be reloaded inplace or not + * @return true if the repository can be reloaded inplace, false otherwise + */ + default boolean isReloadable() { + return false; + } + + /** + * Reload the repository inplace + */ + default void reload(RepositoryMetadata repositoryMetadata) {} + + /** + * Validate the repository metadata + */ + default void validateMetadata(RepositoryMetadata repositoryMetadata) {} } diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 41ad357eaeed9..bf85e19493203 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -297,21 +297,21 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp Setting.Property.NodeScope ); - protected final boolean supportURLRepo; + protected volatile boolean supportURLRepo; - private final int maxShardBlobDeleteBatch; + private volatile int maxShardBlobDeleteBatch; - private final Compressor compressor; + private volatile Compressor compressor; - private final boolean cacheRepositoryData; + private volatile boolean cacheRepositoryData; - private final RateLimiter snapshotRateLimiter; + private volatile RateLimiter snapshotRateLimiter; - private final RateLimiter restoreRateLimiter; + private volatile RateLimiter restoreRateLimiter; - private final RateLimiter remoteUploadRateLimiter; + private volatile RateLimiter remoteUploadRateLimiter; - private final RateLimiter remoteDownloadRateLimiter; + private volatile RateLimiter remoteDownloadRateLimiter; private final CounterMetric snapshotRateLimitingTimeInNanos = new CounterMetric(); @@ -356,7 +356,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp BlobStoreIndexShardSnapshots::fromXContent ); - private final boolean readOnly; + private volatile boolean readOnly; private final boolean isSystemRepository; @@ -366,7 +366,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final SetOnce blobStore = new SetOnce<>(); - private final ClusterService clusterService; + protected final ClusterService clusterService; private final RecoverySettings recoverySettings; @@ -400,36 +400,54 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp /** * IO buffer size hint for reading and writing to the underlying blob store. */ - protected final int bufferSize; + protected volatile int bufferSize; /** * Constructs new BlobStoreRepository - * @param metadata The metadata for this repository including name and settings + * @param repositoryMetadata The metadata for this repository including name and settings * @param clusterService ClusterService */ protected BlobStoreRepository( - final RepositoryMetadata metadata, - final boolean compress, + final RepositoryMetadata repositoryMetadata, final NamedXContentRegistry namedXContentRegistry, final ClusterService clusterService, final RecoverySettings recoverySettings ) { - this.metadata = metadata; + // Read RepositoryMetadata as the first step + readRepositoryMetadata(repositoryMetadata); + + isSystemRepository = SYSTEM_REPOSITORY_SETTING.get(metadata.settings()); this.namedXContentRegistry = namedXContentRegistry; this.threadPool = clusterService.getClusterApplierService().threadPool(); this.clusterService = clusterService; this.recoverySettings = recoverySettings; - this.supportURLRepo = SUPPORT_URL_REPO.get(metadata.settings()); + } + + @Override + public void reload(RepositoryMetadata repositoryMetadata) { + readRepositoryMetadata(repositoryMetadata); + } + + /** + * Reloads the values derived from the Repository Metadata + * + * @param repositoryMetadata RepositoryMetadata instance to derive the values from + */ + private void readRepositoryMetadata(RepositoryMetadata repositoryMetadata) { + this.metadata = repositoryMetadata; + + supportURLRepo = SUPPORT_URL_REPO.get(metadata.settings()); snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB)); restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", ByteSizeValue.ZERO); remoteUploadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_upload_bytes_per_sec", ByteSizeValue.ZERO); remoteDownloadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_download_bytes_per_sec", ByteSizeValue.ZERO); readOnly = READONLY_SETTING.get(metadata.settings()); - isSystemRepository = SYSTEM_REPOSITORY_SETTING.get(metadata.settings()); cacheRepositoryData = CACHE_REPOSITORY_DATA.get(metadata.settings()); bufferSize = Math.toIntExact(BUFFER_SIZE_SETTING.get(metadata.settings()).getBytes()); maxShardBlobDeleteBatch = MAX_SNAPSHOT_SHARD_BLOB_DELETE_BATCH_SIZE.get(metadata.settings()); - this.compressor = compress ? COMPRESSION_TYPE_SETTING.get(metadata.settings()) : CompressorRegistry.none(); + compressor = COMPRESS_SETTING.get(metadata.settings()) + ? COMPRESSION_TYPE_SETTING.get(metadata.settings()) + : CompressorRegistry.none(); } @Override diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java index 54f226e81025e..d4921f4e6d2e7 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -53,13 +53,12 @@ public abstract class MeteredBlobStoreRepository extends BlobStoreRepository { public MeteredBlobStoreRepository( RepositoryMetadata metadata, - boolean compress, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, RecoverySettings recoverySettings, Map location ) { - super(metadata, compress, namedXContentRegistry, clusterService, recoverySettings); + super(metadata, namedXContentRegistry, clusterService, recoverySettings); ThreadPool threadPool = clusterService.getClusterApplierService().threadPool(); this.repositoryInfo = new RepositoryInfo( UUIDs.randomBase64UUID(), @@ -70,6 +69,14 @@ public MeteredBlobStoreRepository( ); } + @Override + public void reload(RepositoryMetadata repositoryMetadata) { + super.reload(repositoryMetadata); + + // Not adding any additional reload logic here is intentional as the constructor only + // initializes the repositoryInfo from the repo metadata, which cannot be changed. + } + public RepositoryStatsSnapshot statsSnapshot() { return new RepositoryStatsSnapshot(repositoryInfo, stats(), RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION, false); } diff --git a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java index d432bab93bd71..4a9a91336ec1d 100644 --- a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java @@ -63,7 +63,6 @@ *
{@code concurrent_streams}
Number of concurrent read/write stream (per repository on each node). Defaults to 5.
*
{@code chunk_size}
Large file can be divided into chunks. This parameter specifies the chunk size. * Defaults to not chucked.
- *
{@code compress}
If set to true metadata files will be stored compressed. Defaults to false.
* * * @opensearch.internal @@ -103,11 +102,11 @@ public class FsRepository extends BlobStoreRepository { public static final Setting BASE_PATH_SETTING = Setting.simpleString("base_path"); - private final Environment environment; + protected final Environment environment; - private ByteSizeValue chunkSize; + protected ByteSizeValue chunkSize; - private final BlobPath basePath; + protected BlobPath basePath; /** * Constructs a shared file system repository. @@ -119,8 +118,27 @@ public FsRepository( ClusterService clusterService, RecoverySettings recoverySettings ) { - super(metadata, calculateCompress(metadata, environment), namedXContentRegistry, clusterService, recoverySettings); + super(metadata, namedXContentRegistry, clusterService, recoverySettings); this.environment = environment; + validateLocation(); + readMetadata(); + } + + protected void readMetadata() { + if (CHUNK_SIZE_SETTING.exists(metadata.settings())) { + this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); + } else { + this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(environment.settings()); + } + final String basePath = BASE_PATH_SETTING.get(metadata.settings()); + if (Strings.hasLength(basePath)) { + this.basePath = new BlobPath().add(basePath); + } else { + this.basePath = BlobPath.cleanPath(); + } + } + + protected void validateLocation() { String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings()); if (location.isEmpty()) { logger.warn( @@ -153,24 +171,6 @@ public FsRepository( ); } } - - if (CHUNK_SIZE_SETTING.exists(metadata.settings())) { - this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); - } else { - this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(environment.settings()); - } - final String basePath = BASE_PATH_SETTING.get(metadata.settings()); - if (Strings.hasLength(basePath)) { - this.basePath = new BlobPath().add(basePath); - } else { - this.basePath = BlobPath.cleanPath(); - } - } - - private static boolean calculateCompress(RepositoryMetadata metadata, Environment environment) { - return COMPRESS_SETTING.exists(metadata.settings()) - ? COMPRESS_SETTING.get(metadata.settings()) - : REPOSITORIES_COMPRESS_SETTING.get(environment.settings()); } @Override diff --git a/server/src/main/java/org/opensearch/repositories/fs/ReloadableFsRepository.java b/server/src/main/java/org/opensearch/repositories/fs/ReloadableFsRepository.java new file mode 100644 index 0000000000000..c06c805a39396 --- /dev/null +++ b/server/src/main/java/org/opensearch/repositories/fs/ReloadableFsRepository.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.fs; + +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.indices.recovery.RecoverySettings; + +/** + * Extension of {@link FsRepository} that can be reloaded inplace + * + * @opensearch.internal + */ +public class ReloadableFsRepository extends FsRepository { + /** + * Constructs a shared file system repository that is reloadable in-place. + */ + public ReloadableFsRepository( + RepositoryMetadata metadata, + Environment environment, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + RecoverySettings recoverySettings + ) { + super(metadata, environment, namedXContentRegistry, clusterService, recoverySettings); + } + + @Override + public boolean isReloadable() { + return true; + } + + @Override + public void reload(RepositoryMetadata repositoryMetadata) { + if (isReloadable() == false) { + return; + } + + super.reload(repositoryMetadata); + validateLocation(); + readMetadata(); + } +} diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index cc4f4f7b6413d..c4599a6e7a00e 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -859,14 +859,7 @@ private static class MeteredRepositoryTypeA extends MeteredBlobStoreRepository { private final TestCryptoProvider cryptoHandler; private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clusterService) { - super( - metadata, - false, - mock(NamedXContentRegistry.class), - clusterService, - mock(RecoverySettings.class), - Map.of("bucket", "bucket-a") - ); + super(metadata, mock(NamedXContentRegistry.class), clusterService, mock(RecoverySettings.class), Map.of("bucket", "bucket-a")); if (metadata.cryptoMetadata() != null) { cryptoHandler = new TestCryptoProvider( @@ -900,14 +893,7 @@ private static class MeteredRepositoryTypeB extends MeteredBlobStoreRepository { private final TestCryptoProvider cryptoHandler; private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clusterService) { - super( - metadata, - false, - mock(NamedXContentRegistry.class), - clusterService, - mock(RecoverySettings.class), - Map.of("bucket", "bucket-b") - ); + super(metadata, mock(NamedXContentRegistry.class), clusterService, mock(RecoverySettings.class), Map.of("bucket", "bucket-b")); if (metadata.cryptoMetadata() != null) { cryptoHandler = new TestCryptoProvider( diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index e097c7025e4fe..9c65ad32fa6a6 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -252,7 +252,7 @@ public void testBadChunksize() throws Exception { ); } - public void testFsRepositoryCompressDeprecated() { + public void testFsRepositoryCompressDeprecatedIgnored() { final Path location = OpenSearchIntegTestCase.randomRepoPath(node().settings()); final Settings settings = Settings.builder().put(node().settings()).put("location", location).build(); final RepositoryMetadata metadata = new RepositoryMetadata("test-repo", REPO_TYPE, settings); @@ -265,10 +265,7 @@ public void testFsRepositoryCompressDeprecated() { new FsRepository(metadata, useCompressEnvironment, null, BlobStoreTestUtil.mockClusterService(), null); - assertWarnings( - "[repositories.fs.compress] setting was deprecated in OpenSearch and will be removed in a future release!" - + " See the breaking changes documentation for the next major version." - ); + assertNoDeprecationWarnings(); } private static void writeIndexGen(BlobStoreRepository repository, RepositoryData repositoryData, long generation) throws Exception { diff --git a/server/src/test/java/org/opensearch/repositories/fs/ReloadableFsRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/fs/ReloadableFsRepositoryTests.java new file mode 100644 index 0000000000000..db2cf9c3e9582 --- /dev/null +++ b/server/src/test/java/org/opensearch/repositories/fs/ReloadableFsRepositoryTests.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.fs; + +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.compress.ZstdCompressor; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.compress.CompressorRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.repositories.blobstore.BlobStoreTestUtil; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.file.Path; +import java.util.Locale; + +public class ReloadableFsRepositoryTests extends OpenSearchTestCase { + ReloadableFsRepository repository; + RepositoryMetadata metadata; + Settings settings; + Path repo; + + @Override + public void setUp() throws Exception { + super.setUp(); + + repo = createTempDir(); + settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()) + .put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()) + .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()) + .put("location", repo) + .put("compress", false) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(FsRepository.BASE_PATH_SETTING.getKey(), "my_base_path") + .build(); + metadata = new RepositoryMetadata("test", "fs", settings); + repository = new ReloadableFsRepository( + metadata, + new Environment(settings, null), + NamedXContentRegistry.EMPTY, + BlobStoreTestUtil.mockClusterService(), + new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + } + + /** + * Validates that {@link ReloadableFsRepository} supports inplace reloading + */ + public void testIsReloadable() { + assertTrue(repository.isReloadable()); + } + + /** + * Updates repository metadata of an existing repository to enable default compressor + */ + public void testCompressReload() { + assertEquals(CompressorRegistry.none(), repository.getCompressor()); + updateCompressionTypeToDefault(); + repository.validateMetadata(metadata); + repository.reload(metadata); + assertEquals(CompressorRegistry.defaultCompressor(), repository.getCompressor()); + } + + /** + * Updates repository metadata of an existing repository to change compressor type from default to Zstd + */ + public void testCompressionTypeReload() { + assertEquals(CompressorRegistry.none(), repository.getCompressor()); + updateCompressionTypeToDefault(); + repository = new ReloadableFsRepository( + metadata, + new Environment(settings, null), + NamedXContentRegistry.EMPTY, + BlobStoreTestUtil.mockClusterService(), + new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + assertEquals(CompressorRegistry.defaultCompressor(), repository.getCompressor()); + + settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()) + .put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()) + .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()) + .put("location", repo) + .put("compress", true) + .put("compression_type", ZstdCompressor.NAME.toLowerCase(Locale.ROOT)) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(FsRepository.BASE_PATH_SETTING.getKey(), "my_base_path") + .build(); + metadata = new RepositoryMetadata("test", "fs", settings); + repository.validateMetadata(metadata); + repository.reload(metadata); + assertEquals(CompressorRegistry.getCompressor(ZstdCompressor.NAME.toUpperCase(Locale.ROOT)), repository.getCompressor()); + } + + private void updateCompressionTypeToDefault() { + settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()) + .put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()) + .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()) + .put("location", repo) + .put("compress", true) + .put("compression_type", DeflateCompressor.NAME.toLowerCase(Locale.ROOT)) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(FsRepository.BASE_PATH_SETTING.getKey(), "my_base_path") + .build(); + metadata = new RepositoryMetadata("test", "fs", settings); + } +} diff --git a/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java b/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java index ca8bec469f3bc..f9388c9e4b86e 100644 --- a/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java +++ b/server/src/test/java/org/opensearch/snapshots/mockstore/MockEventuallyConsistentRepository.java @@ -90,7 +90,7 @@ public MockEventuallyConsistentRepository( final Context context, final Random random ) { - super(metadata, false, namedXContentRegistry, clusterService, recoverySettings); + super(metadata, namedXContentRegistry, clusterService, recoverySettings); this.context = context; this.namedXContentRegistry = namedXContentRegistry; this.random = random; From 71b69482a82b647bc6b1bfbc011c9ea96511f1e4 Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Wed, 4 Oct 2023 18:35:24 -0700 Subject: [PATCH 110/388] Adding cancellation start time in client side TaskInfo (#10326) Signed-off-by: Sagar Upadhyaya --- .../org/opensearch/client/tasks/TaskInfo.java | 18 ++++++++++++++++-- .../client/tasks/CancelTasksResponseTests.java | 8 +++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskInfo.java b/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskInfo.java index 51ac62830446f..75badc4e3dbf2 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskInfo.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskInfo.java @@ -54,6 +54,7 @@ public class TaskInfo { private long runningTimeNanos; private boolean cancellable; private boolean cancelled; + private Long cancellationStartTime; private TaskId parentTaskId; private final Map status = new HashMap<>(); private final Map headers = new HashMap<>(); @@ -127,6 +128,14 @@ void setCancelled(boolean cancelled) { this.cancelled = cancelled; } + public Long getCancellationStartTime() { + return this.cancellationStartTime; + } + + public void setCancellationStartTime(Long cancellationStartTime) { + this.cancellationStartTime = cancellationStartTime; + } + public TaskId getParentTaskId() { return parentTaskId; } @@ -180,6 +189,7 @@ private void noOpParse(Object s) {} parser.declareString(TaskInfo::setParentTaskId, new ParseField("parent_task_id")); parser.declareObject(TaskInfo::setHeaders, (p, c) -> p.mapStrings(), new ParseField("headers")); parser.declareObject(TaskInfo::setResourceStats, (p, c) -> p.map(), new ParseField("resource_stats")); + parser.declareLong(TaskInfo::setCancellationStartTime, new ParseField("cancellation_time_millis")); PARSER = (XContentParser p, Void v, String name) -> parser.parse(p, new TaskInfo(new TaskId(name)), null); } @@ -199,7 +209,8 @@ && isCancelled() == taskInfo.isCancelled() && Objects.equals(getParentTaskId(), taskInfo.getParentTaskId()) && Objects.equals(status, taskInfo.status) && Objects.equals(getHeaders(), taskInfo.getHeaders()) - && Objects.equals(getResourceStats(), taskInfo.getResourceStats()); + && Objects.equals(getResourceStats(), taskInfo.getResourceStats()) + && Objects.equals(getCancellationStartTime(), taskInfo.cancellationStartTime); } @Override @@ -216,7 +227,8 @@ public int hashCode() { getParentTaskId(), status, getHeaders(), - getResourceStats() + getResourceStats(), + getCancellationStartTime() ); } @@ -250,6 +262,8 @@ public String toString() { + headers + ", resource_stats=" + resourceStats + + ", cancellationStartTime=" + + cancellationStartTime + '}'; } } diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java index 835a93b5b09ce..faf5024d0c173 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java @@ -84,6 +84,10 @@ protected CancelTasksResponseTests.ByNodeCancelTasksResponse createServerTestIns for (int i = 0; i < 4; i++) { boolean cancellable = randomBoolean(); boolean cancelled = cancellable == true ? randomBoolean() : false; + Long cancellationStartTime = null; + if (cancelled) { + cancellationStartTime = randomNonNegativeLong(); + } tasks.add( new org.opensearch.tasks.TaskInfo( new TaskId(NODE_ID, (long) i), @@ -97,7 +101,8 @@ protected CancelTasksResponseTests.ByNodeCancelTasksResponse createServerTestIns cancelled, new TaskId("node1", randomLong()), Collections.singletonMap("x-header-of", "some-value"), - null + null, + cancellationStartTime ) ); } @@ -135,6 +140,7 @@ protected void assertInstances( assertEquals(ti.isCancelled(), taskInfo.isCancelled()); assertEquals(ti.getParentTaskId().getNodeId(), taskInfo.getParentTaskId().getNodeId()); assertEquals(ti.getParentTaskId().getId(), taskInfo.getParentTaskId().getId()); + assertEquals(ti.getCancellationStartTime(), taskInfo.getCancellationStartTime()); FakeTaskStatus status = (FakeTaskStatus) ti.getStatus(); assertEquals(status.code, taskInfo.getStatus().get("code")); assertEquals(status.status, taskInfo.getStatus().get("status")); From 3dd9fc1064a079b593a3fc9a781f7bc5be3a0096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 18:47:21 -0700 Subject: [PATCH 111/388] Bump org.apache.commons:commons-compress in /buildSrc (#10205) Bumps org.apache.commons:commons-compress from 1.23.0 to 1.24.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-compress dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- buildSrc/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index d7bdd09ea882e..ccf6a6010ff63 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -103,7 +103,7 @@ dependencies { api localGroovy() api 'commons-codec:commons-codec:1.16.0' - api 'org.apache.commons:commons-compress:1.23.0' + api 'org.apache.commons:commons-compress:1.24.0' api 'org.apache.ant:ant:1.10.14' api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0' api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0' From 211578ce8ea0259d23d823652ba549bb7e66350e Mon Sep 17 00:00:00 2001 From: Gagan Juneja Date: Thu, 5 Oct 2023 13:29:55 +0530 Subject: [PATCH 112/388] Add Telemetry metrics framework (#10241) Signed-off-by: Gagan Juneja --- CHANGELOG.md | 3 +- .../opensearch/telemetry/metrics/Counter.java | 35 +++ .../metrics/DefaultMetricsRegistry.java | 41 ++++ .../telemetry/metrics/MetricsRegistry.java | 39 ++++ .../telemetry/metrics/MetricsTelemetry.java | 4 +- .../telemetry/metrics/noop/NoopCounter.java | 38 ++++ .../metrics/noop/NoopMetricsRegistry.java | 45 ++++ .../telemetry/metrics/noop/package-info.java | 16 ++ .../telemetry/metrics/tags/Tags.java | 99 +++++++++ .../telemetry/metrics/tags/package-info.java | 16 ++ .../telemetry/tracing/TracingTelemetry.java | 5 - .../metrics/DefaultMetricsRegistryTests.java | 51 +++++ plugins/telemetry-otel/build.gradle | 22 +- .../config/telemetry-otel/log4j2.properties | 20 ++ ...-extension-incubator-1.30.1-alpha.jar.sha1 | 1 + ...ntelemetry-extension-incubator-LICENSE.txt | 202 ++++++++++++++++++ ...entelemetry-extension-incubator-NOTICE.txt | 0 .../OTelAttributesConverter.java | 21 +- .../telemetry/OTelTelemetryPlugin.java | 19 +- .../telemetry/OTelTelemetrySettings.java | 27 +++ .../telemetry/metrics/OTelCounter.java | 40 ++++ .../metrics/OTelMetricsTelemetry.java | 67 ++++++ .../telemetry/metrics/OTelUpDownCounter.java | 40 ++++ .../exporter/OTelMetricsExporterFactory.java | 90 ++++++++ .../metrics/exporter/package-info.java | 12 ++ .../telemetry/metrics/package-info.java | 12 ++ .../tracing/OTelResourceProvider.java | 49 ++++- .../tracing/OTelTracingTelemetry.java | 27 ++- .../plugin-metadata/plugin-security.policy | 1 + .../telemetry/OTelTelemetryPluginTests.java | 24 ++- .../metrics/OTelMetricsTelemetryTests.java | 107 ++++++++++ .../metrics/exporter/DummyMetricExporter.java | 39 ++++ .../OTelMetricsExporterFactoryTests.java | 78 +++++++ .../tracing/OTelAttributesConverterTests.java | 14 +- .../tracing/OTelTracingTelemetryTests.java | 31 +-- .../main/java/org/opensearch/node/Node.java | 12 ++ .../telemetry/TelemetrySettings.java | 11 + .../metrics/MetricsRegistryFactory.java | 76 +++++++ .../metrics/NoopMetricsRegistryFactory.java | 36 ++++ .../telemetry/metrics/package-info.java | 12 ++ .../metrics/MetricsRegistryFactoryTests.java | 71 ++++++ .../test/telemetry/MockTelemetry.java | 15 ++ 42 files changed, 1489 insertions(+), 79 deletions(-) create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/Counter.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistry.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistry.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopCounter.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopMetricsRegistry.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/package-info.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/Tags.java create mode 100644 libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/package-info.java create mode 100644 libs/telemetry/src/test/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistryTests.java create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-LICENSE.txt create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-NOTICE.txt rename plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/{tracing => }/OTelAttributesConverter.java (71%) create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelCounter.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelUpDownCounter.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactory.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/package-info.java create mode 100644 plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/package-info.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/DummyMetricExporter.java create mode 100644 plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactoryTests.java create mode 100644 server/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistryFactory.java create mode 100644 server/src/main/java/org/opensearch/telemetry/metrics/NoopMetricsRegistryFactory.java create mode 100644 server/src/main/java/org/opensearch/telemetry/metrics/package-info.java create mode 100644 server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index aea4a45b60ffc..153cc12c5f9bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add instrumentation in Inbound Handler. ([#100143](https://github.com/opensearch-project/OpenSearch/pull/10143)) - Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356)) - [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569)) +- [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241)) ### Deprecated @@ -147,4 +148,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/Counter.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/Counter.java new file mode 100644 index 0000000000000..c62288d280e2f --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/Counter.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.telemetry.metrics.tags.Tags; + +/** + * Counter adds the value to the existing metric. + * {@opensearch.experimental} + */ +@ExperimentalApi +public interface Counter { + + /** + * add value. + * @param value value to be added. + */ + void add(double value); + + /** + * add value along with the attributes. + * + * @param value value to be added. + * @param tags attributes/dimensions of the metric. + */ + void add(double value, Tags tags); + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistry.java new file mode 100644 index 0000000000000..d57def9406b17 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistry.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import java.io.IOException; + +/** + * Default implementation for {@link MetricsRegistry} + */ +class DefaultMetricsRegistry implements MetricsRegistry { + private final MetricsTelemetry metricsTelemetry; + + /** + * Constructor + * @param metricsTelemetry metrics telemetry. + */ + public DefaultMetricsRegistry(MetricsTelemetry metricsTelemetry) { + this.metricsTelemetry = metricsTelemetry; + } + + @Override + public Counter createCounter(String name, String description, String unit) { + return metricsTelemetry.createCounter(name, description, unit); + } + + @Override + public Counter createUpDownCounter(String name, String description, String unit) { + return metricsTelemetry.createUpDownCounter(name, description, unit); + } + + @Override + public void close() throws IOException { + metricsTelemetry.close(); + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistry.java new file mode 100644 index 0000000000000..61b3df089928b --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistry.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.Closeable; + +/** + * MetricsRegistry helps in creating the metric instruments. + * @opensearch.experimental + */ +@ExperimentalApi +public interface MetricsRegistry extends Closeable { + + /** + * Creates the counter. + * @param name name of the counter. + * @param description any description about the metric. + * @param unit unit of the metric. + * @return counter. + */ + Counter createCounter(String name, String description, String unit); + + /** + * Creates the upDown counter. + * @param name name of the upDown counter. + * @param description any description about the metric. + * @param unit unit of the metric. + * @return counter. + */ + Counter createUpDownCounter(String name, String description, String unit); +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java index 0bf9482fe58d8..2f70c28efb1cd 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java @@ -10,12 +10,14 @@ import org.opensearch.common.annotation.ExperimentalApi; +import java.io.Closeable; + /** * Interface for metrics telemetry providers * * @opensearch.experimental */ @ExperimentalApi -public interface MetricsTelemetry { +public interface MetricsTelemetry extends MetricsRegistry, Closeable { } diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopCounter.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopCounter.java new file mode 100644 index 0000000000000..c1daf564dd3bc --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopCounter.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics.noop; + +import org.opensearch.common.annotation.InternalApi; +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.tags.Tags; + +/** + * No-op {@link Counter} + * {@opensearch.internal} + */ +@InternalApi +public class NoopCounter implements Counter { + + /** + * No-op Counter instance + */ + public final static NoopCounter INSTANCE = new NoopCounter(); + + private NoopCounter() {} + + @Override + public void add(double value) { + + } + + @Override + public void add(double value, Tags tags) { + + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopMetricsRegistry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopMetricsRegistry.java new file mode 100644 index 0000000000000..640c6842a8960 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/NoopMetricsRegistry.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics.noop; + +import org.opensearch.common.annotation.InternalApi; +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.MetricsRegistry; + +import java.io.IOException; + +/** + *No-op {@link MetricsRegistry} + * {@opensearch.internal} + */ +@InternalApi +public class NoopMetricsRegistry implements MetricsRegistry { + + /** + * No-op Meter instance + */ + public final static NoopMetricsRegistry INSTANCE = new NoopMetricsRegistry(); + + private NoopMetricsRegistry() {} + + @Override + public Counter createCounter(String name, String description, String unit) { + return NoopCounter.INSTANCE; + } + + @Override + public Counter createUpDownCounter(String name, String description, String unit) { + return NoopCounter.INSTANCE; + } + + @Override + public void close() throws IOException { + + } +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/package-info.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/package-info.java new file mode 100644 index 0000000000000..7c7ed08044993 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/noop/package-info.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains metrics related classes + * {@opensearch.internal} + */ +@InternalApi +package org.opensearch.telemetry.metrics.noop; + +import org.opensearch.common.annotation.InternalApi; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/Tags.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/Tags.java new file mode 100644 index 0000000000000..f2a8764f8021d --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/Tags.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics.tags; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Class to create tags for a meter. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Tags { + private final Map tagsMap; + /** + * Empty value. + */ + public final static Tags EMPTY = new Tags(Collections.emptyMap()); + + /** + * Factory method. + * @return tags. + */ + public static Tags create() { + return new Tags(new HashMap<>()); + } + + /** + * Constructor. + */ + private Tags(Map tagsMap) { + this.tagsMap = tagsMap; + } + + /** + * Add String attribute. + * @param key key + * @param value value + * @return Same instance. + */ + public Tags addTag(String key, String value) { + Objects.requireNonNull(value, "value cannot be null"); + tagsMap.put(key, value); + return this; + } + + /** + * Add long attribute. + * @param key key + * @param value value + * @return Same instance. + */ + public Tags addTag(String key, long value) { + tagsMap.put(key, value); + return this; + }; + + /** + * Add double attribute. + * @param key key + * @param value value + * @return Same instance. + */ + public Tags addTag(String key, double value) { + tagsMap.put(key, value); + return this; + }; + + /** + * Add boolean attribute. + * @param key key + * @param value value + * @return Same instance. + */ + public Tags addTag(String key, boolean value) { + tagsMap.put(key, value); + return this; + }; + + /** + * Returns the attribute map. + * @return tags map + */ + public Map getTagsMap() { + return Collections.unmodifiableMap(tagsMap); + } + +} diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/package-info.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/package-info.java new file mode 100644 index 0000000000000..70bc9be992b32 --- /dev/null +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/tags/package-info.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains metrics related classes + * @opensearch.experimental + */ +@ExperimentalApi +package org.opensearch.telemetry.metrics.tags; + +import org.opensearch.common.annotation.ExperimentalApi; diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java index 38c9104c3ec35..f04a505088424 100644 --- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java +++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/TracingTelemetry.java @@ -35,9 +35,4 @@ public interface TracingTelemetry extends Closeable { */ TracingContextPropagator getContextPropagator(); - /** - * closes the resource - */ - void close(); - } diff --git a/libs/telemetry/src/test/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistryTests.java b/libs/telemetry/src/test/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistryTests.java new file mode 100644 index 0000000000000..6171641db5f07 --- /dev/null +++ b/libs/telemetry/src/test/java/org/opensearch/telemetry/metrics/DefaultMetricsRegistryTests.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.test.OpenSearchTestCase; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DefaultMetricsRegistryTests extends OpenSearchTestCase { + + private MetricsTelemetry metricsTelemetry; + private DefaultMetricsRegistry defaultMeterRegistry; + + @Override + public void setUp() throws Exception { + super.setUp(); + metricsTelemetry = mock(MetricsTelemetry.class); + defaultMeterRegistry = new DefaultMetricsRegistry(metricsTelemetry); + } + + public void testCounter() { + Counter mockCounter = mock(Counter.class); + when(defaultMeterRegistry.createCounter(any(String.class), any(String.class), any(String.class))).thenReturn(mockCounter); + Counter counter = defaultMeterRegistry.createCounter( + "org.opensearch.telemetry.metrics.DefaultMeterRegistryTests.testCounter", + "test counter", + "1" + ); + assertSame(mockCounter, counter); + } + + public void testUpDownCounter() { + Counter mockCounter = mock(Counter.class); + when(defaultMeterRegistry.createUpDownCounter(any(String.class), any(String.class), any(String.class))).thenReturn(mockCounter); + Counter counter = defaultMeterRegistry.createUpDownCounter( + "org.opensearch.telemetry.metrics.DefaultMeterRegistryTests.testUpDownCounter", + "test up-down counter", + "1" + ); + assertSame(mockCounter, counter); + } + +} diff --git a/plugins/telemetry-otel/build.gradle b/plugins/telemetry-otel/build.gradle index 45c9f522c09d8..04fff20947b4f 100644 --- a/plugins/telemetry-otel/build.gradle +++ b/plugins/telemetry-otel/build.gradle @@ -37,6 +37,7 @@ dependencies { runtimeOnly "com.squareup.okhttp3:okhttp:4.11.0" runtimeOnly "com.squareup.okio:okio-jvm:3.5.0" runtimeOnly "io.opentelemetry:opentelemetry-exporter-sender-okhttp:${versions.opentelemetry}" + api "io.opentelemetry:opentelemetry-extension-incubator:${versions.opentelemetry}-alpha" testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${versions.opentelemetry}" } @@ -80,29 +81,12 @@ thirdPartyAudit { 'io.opentelemetry.api.events.EventEmitter', 'io.opentelemetry.api.events.EventEmitterBuilder', 'io.opentelemetry.api.events.EventEmitterProvider', - 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleHistogramBuilder', - 'io.opentelemetry.extension.incubator.metrics.ExtendedLongHistogramBuilder', 'io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties', 'io.opentelemetry.sdk.autoconfigure.spi.logs.ConfigurableLogRecordExporterProvider', 'io.opentelemetry.sdk.autoconfigure.spi.metrics.ConfigurableMetricExporterProvider', 'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider', - 'io.opentelemetry.extension.incubator.metrics.DoubleCounterAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.DoubleGauge', - 'io.opentelemetry.extension.incubator.metrics.DoubleGaugeAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.DoubleHistogramAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.DoubleUpDownCounterAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleCounterBuilder', - 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleGaugeBuilder', - 'io.opentelemetry.extension.incubator.metrics.ExtendedDoubleUpDownCounterBuilder', - 'io.opentelemetry.extension.incubator.metrics.ExtendedLongCounterBuilder', - 'io.opentelemetry.extension.incubator.metrics.ExtendedLongGaugeBuilder', - 'io.opentelemetry.extension.incubator.metrics.ExtendedLongUpDownCounterBuilder', - 'io.opentelemetry.extension.incubator.metrics.LongCounterAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.LongGauge', - 'io.opentelemetry.extension.incubator.metrics.LongGaugeAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.LongHistogramAdviceConfigurer', - 'io.opentelemetry.extension.incubator.metrics.LongUpDownCounterAdviceConfigurer', - 'kotlin.io.path.PathsKt' + 'kotlin.io.path.PathsKt', + 'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider' ) } diff --git a/plugins/telemetry-otel/config/telemetry-otel/log4j2.properties b/plugins/telemetry-otel/config/telemetry-otel/log4j2.properties index 544f42bd5513b..8dec1119eec66 100644 --- a/plugins/telemetry-otel/config/telemetry-otel/log4j2.properties +++ b/plugins/telemetry-otel/config/telemetry-otel/log4j2.properties @@ -25,3 +25,23 @@ logger.exporter.name = io.opentelemetry.exporter.logging.LoggingSpanExporter logger.exporter.level = INFO logger.exporter.appenderRef.tracing.ref = tracing logger.exporter.additivity = false + + +appender.metrics.type = RollingFile +appender.metrics.name = metrics +appender.metrics.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_otel_metrics.log +appender.metrics.filePermissions = rw-r----- +appender.metrics.layout.type = PatternLayout +appender.metrics.layout.pattern = %m%n +appender.metrics.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_otel_metrics-%i.log.gz +appender.metrics.policies.type = Policies +appender.metrics.policies.size.type = SizeBasedTriggeringPolicy +appender.metrics.policies.size.size = 1GB +appender.metrics.strategy.type = DefaultRolloverStrategy +appender.metrics.strategy.max = 4 + + +logger.metrics_exporter.name = io.opentelemetry.exporter.logging.LoggingMetricExporter +logger.metrics_exporter.level = INFO +logger.metrics_exporter.appenderRef.tracing.ref = metrics +logger.metrics_exporter.additivity = false diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1 new file mode 100644 index 0000000000000..bde43937e82e4 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1 @@ -0,0 +1 @@ +bfcea9bd71f97dd4e8a4f92c15ba5659fb07ff05 \ No newline at end of file diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-LICENSE.txt b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-NOTICE.txt b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelAttributesConverter.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelAttributesConverter.java similarity index 71% rename from plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelAttributesConverter.java rename to plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelAttributesConverter.java index 4d0966e6b5185..98d265e92ba3c 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelAttributesConverter.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelAttributesConverter.java @@ -6,7 +6,9 @@ * compatible open source license. */ -package org.opensearch.telemetry.tracing; +package org.opensearch.telemetry; + +import org.opensearch.telemetry.metrics.tags.Tags; import java.util.Locale; @@ -16,7 +18,7 @@ /** * Converts {@link org.opensearch.telemetry.tracing.attributes.Attributes} to OTel {@link Attributes} */ -final class OTelAttributesConverter { +public final class OTelAttributesConverter { /** * Constructor. @@ -28,7 +30,7 @@ private OTelAttributesConverter() {} * @param attributes attributes * @return otel attributes. */ - static Attributes convert(org.opensearch.telemetry.tracing.attributes.Attributes attributes) { + public static Attributes convert(org.opensearch.telemetry.tracing.attributes.Attributes attributes) { AttributesBuilder attributesBuilder = Attributes.builder(); if (attributes != null) { attributes.getAttributesMap().forEach((x, y) -> addSpanAttribute(x, y, attributesBuilder)); @@ -49,4 +51,17 @@ private static void addSpanAttribute(String key, Object value, AttributesBuilder throw new IllegalArgumentException(String.format(Locale.ROOT, "Span attribute value %s type not supported", value)); } } + + /** + * Attribute converter. + * @param tags attributes + * @return otel attributes. + */ + public static Attributes convert(Tags tags) { + AttributesBuilder attributesBuilder = Attributes.builder(); + if (tags != null) { + tags.getTagsMap().forEach((x, y) -> addSpanAttribute(x, y, attributesBuilder)); + } + return attributesBuilder.build(); + } } diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java index 1af88196e3727..b57876c9310f3 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java @@ -12,7 +12,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.TelemetryPlugin; -import org.opensearch.telemetry.metrics.MetricsTelemetry; +import org.opensearch.telemetry.metrics.OTelMetricsTelemetry; import org.opensearch.telemetry.tracing.OTelResourceProvider; import org.opensearch.telemetry.tracing.OTelTelemetry; import org.opensearch.telemetry.tracing.OTelTracingTelemetry; @@ -21,11 +21,18 @@ import java.util.List; import java.util.Optional; +import io.opentelemetry.sdk.OpenTelemetrySdk; + /** * Telemetry plugin based on Otel */ public class OTelTelemetryPlugin extends Plugin implements TelemetryPlugin { + /** + * Instrumentation scope name. + */ + public static final String INSTRUMENTATION_SCOPE_NAME = "org.opensearch.telemetry"; + static final String OTEL_TRACER_NAME = "otel"; private final Settings settings; @@ -44,7 +51,8 @@ public List> getSettings() { OTelTelemetrySettings.TRACER_EXPORTER_BATCH_SIZE_SETTING, OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING, OTelTelemetrySettings.TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING, - OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING + OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING, + OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING ); } @@ -59,8 +67,11 @@ public String getName() { } private Telemetry telemetry(TelemetrySettings telemetrySettings) { - return new OTelTelemetry(new OTelTracingTelemetry(OTelResourceProvider.get(telemetrySettings, settings)), new MetricsTelemetry() { - }); + final OpenTelemetrySdk openTelemetry = OTelResourceProvider.get(telemetrySettings, settings); + return new OTelTelemetry( + new OTelTracingTelemetry<>(openTelemetry, openTelemetry.getSdkTracerProvider()), + new OTelMetricsTelemetry<>(openTelemetry.getSdkMeterProvider()) + ); } } diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java index 59c87cca22986..8e23f724b4570 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetrySettings.java @@ -11,13 +11,16 @@ import org.opensearch.SpecialPermission; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; +import org.opensearch.telemetry.metrics.exporter.OTelMetricsExporterFactory; import org.opensearch.telemetry.tracing.exporter.OTelSpanExporterFactory; import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import io.opentelemetry.exporter.logging.LoggingMetricExporter; import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.trace.export.SpanExporter; /** @@ -83,4 +86,28 @@ private OTelTelemetrySettings() {} Setting.Property.NodeScope, Setting.Property.Final ); + + /** + * Metrics Exporter type setting. + */ + @SuppressWarnings("unchecked") + public static final Setting> OTEL_METRICS_EXPORTER_CLASS_SETTING = new Setting<>( + "telemetry.otel.metrics.exporter.class", + LoggingMetricExporter.class.getName(), + className -> { + // Check we ourselves are not being called by unprivileged code. + SpecialPermission.check(); + + try { + return AccessController.doPrivileged((PrivilegedExceptionAction>) () -> { + final ClassLoader loader = OTelMetricsExporterFactory.class.getClassLoader(); + return (Class) loader.loadClass(className); + }); + } catch (PrivilegedActionException ex) { + throw new IllegalStateException("Unable to load span exporter class:" + className, ex.getCause()); + } + }, + Setting.Property.NodeScope, + Setting.Property.Final + ); } diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelCounter.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelCounter.java new file mode 100644 index 0000000000000..b72f63e027243 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelCounter.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.telemetry.OTelAttributesConverter; +import org.opensearch.telemetry.metrics.tags.Tags; + +import io.opentelemetry.api.metrics.DoubleCounter; + +/** + * OTel Counter + */ +class OTelCounter implements Counter { + + private final DoubleCounter otelDoubleCounter; + + /** + * Constructor + * @param otelDoubleCounter delegate counter. + */ + public OTelCounter(DoubleCounter otelDoubleCounter) { + this.otelDoubleCounter = otelDoubleCounter; + } + + @Override + public void add(double value) { + otelDoubleCounter.add(value); + } + + @Override + public void add(double value, Tags tags) { + otelDoubleCounter.add(value, OTelAttributesConverter.convert(tags)); + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java new file mode 100644 index 0000000000000..8598e5976d20d --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.telemetry.OTelTelemetryPlugin; + +import java.io.Closeable; +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; + +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; + +/** + * OTel implementation for {@link MetricsTelemetry} + */ +public class OTelMetricsTelemetry implements MetricsTelemetry { + private final Meter otelMeter; + private final T meterProvider; + + /** + * Creates OTel based {@link MetricsTelemetry}. + * @param meterProvider {@link MeterProvider} instance + */ + public OTelMetricsTelemetry(T meterProvider) { + this.meterProvider = meterProvider; + this.otelMeter = meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME); + } + + @Override + public Counter createCounter(String name, String description, String unit) { + DoubleCounter doubleCounter = AccessController.doPrivileged( + (PrivilegedAction) () -> otelMeter.counterBuilder(name) + .setUnit(unit) + .setDescription(description) + .ofDoubles() + .build() + ); + return new OTelCounter(doubleCounter); + } + + @Override + public Counter createUpDownCounter(String name, String description, String unit) { + DoubleUpDownCounter doubleUpDownCounter = AccessController.doPrivileged( + (PrivilegedAction) () -> otelMeter.upDownCounterBuilder(name) + .setUnit(unit) + .setDescription(description) + .ofDoubles() + .build() + ); + return new OTelUpDownCounter(doubleUpDownCounter); + } + + @Override + public void close() throws IOException { + meterProvider.close(); + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelUpDownCounter.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelUpDownCounter.java new file mode 100644 index 0000000000000..2f40881996f7e --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelUpDownCounter.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.telemetry.OTelAttributesConverter; +import org.opensearch.telemetry.metrics.tags.Tags; + +import io.opentelemetry.api.metrics.DoubleUpDownCounter; + +/** + * OTel Counter + */ +public class OTelUpDownCounter implements Counter { + + private final DoubleUpDownCounter doubleUpDownCounter; + + /** + * Constructor + * @param doubleUpDownCounter delegate counter. + */ + public OTelUpDownCounter(DoubleUpDownCounter doubleUpDownCounter) { + this.doubleUpDownCounter = doubleUpDownCounter; + } + + @Override + public void add(double value) { + doubleUpDownCounter.add(value); + } + + @Override + public void add(double value, Tags tags) { + doubleUpDownCounter.add(value, OTelAttributesConverter.convert(tags)); + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactory.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactory.java new file mode 100644 index 0000000000000..ef5a31e4003ca --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactory.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics.exporter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.SpecialPermission; +import org.opensearch.common.settings.Settings; +import org.opensearch.telemetry.OTelTelemetrySettings; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.reflect.Method; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; + +import io.opentelemetry.sdk.metrics.export.MetricExporter; + +/** + * Factory class to create the {@link MetricExporter} instance. + */ +public class OTelMetricsExporterFactory { + + private static final Logger logger = LogManager.getLogger(OTelMetricsExporterFactory.class); + + /** + * Base constructor. + */ + private OTelMetricsExporterFactory() { + + } + + /** + * Creates the {@link MetricExporter} instances based on the OTEL_METRIC_EXPORTER_CLASS_SETTING value. + * As of now, it expects the MetricExporter implementations to have a create factory method to instantiate the + * MetricExporter. + * @param settings settings. + * @return MetricExporter instance. + */ + public static MetricExporter create(Settings settings) { + Class MetricExporterProviderClass = OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.get(settings); + MetricExporter metricExporter = instantiateExporter(MetricExporterProviderClass); + logger.info("Successfully instantiated the Metrics MetricExporter class {}", MetricExporterProviderClass); + return metricExporter; + } + + private static MetricExporter instantiateExporter(Class exporterProviderClass) { + try { + // Check we ourselves are not being called by unprivileged code. + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + String methodName = "create"; + String getDefaultMethod = "getDefault"; + for (Method m : exporterProviderClass.getMethods()) { + if (m.getName().equals(getDefaultMethod)) { + methodName = getDefaultMethod; + break; + } + } + try { + return (MetricExporter) MethodHandles.publicLookup() + .findStatic(exporterProviderClass, methodName, MethodType.methodType(exporterProviderClass)) + .asType(MethodType.methodType(MetricExporter.class)) + .invokeExact(); + } catch (Throwable e) { + if (e.getCause() instanceof NoSuchMethodException) { + throw new IllegalStateException("No create factory method exist in [" + exporterProviderClass.getName() + "]"); + } else { + throw new IllegalStateException( + "MetricExporter instantiation failed for class [" + exporterProviderClass.getName() + "]", + e.getCause() + ); + } + } + }); + } catch (PrivilegedActionException ex) { + throw new IllegalStateException( + "MetricExporter instantiation failed for class [" + exporterProviderClass.getName() + "]", + ex.getCause() + ); + } + } +} diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/package-info.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/package-info.java new file mode 100644 index 0000000000000..b48ec3e2336c4 --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/exporter/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for tracing requests. + */ +package org.opensearch.telemetry.metrics.exporter; diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/package-info.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/package-info.java new file mode 100644 index 0000000000000..803c159eb201a --- /dev/null +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for tracing requests. + */ +package org.opensearch.telemetry.metrics; diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java index fe05cc8bb7a41..a6a1f12aab8a9 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java @@ -10,6 +10,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.metrics.exporter.OTelMetricsExporterFactory; import org.opensearch.telemetry.tracing.exporter.OTelSpanExporterFactory; import org.opensearch.telemetry.tracing.sampler.ProbabilisticSampler; import org.opensearch.telemetry.tracing.sampler.RequestSampler; @@ -18,11 +19,12 @@ import java.security.PrivilegedAction; import java.util.concurrent.TimeUnit; -import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; import io.opentelemetry.context.propagation.ContextPropagators; import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; @@ -44,11 +46,11 @@ private OTelResourceProvider() {} * Creates OpenTelemetry instance with default configuration * @param telemetrySettings telemetry settings * @param settings cluster settings - * @return OpenTelemetry instance + * @return OpenTelemetrySdk instance */ - public static OpenTelemetry get(TelemetrySettings telemetrySettings, Settings settings) { + public static OpenTelemetrySdk get(TelemetrySettings telemetrySettings, Settings settings) { return AccessController.doPrivileged( - (PrivilegedAction) () -> get( + (PrivilegedAction) () -> get( settings, OTelSpanExporterFactory.create(settings), ContextPropagators.create(W3CTraceContextPropagator.getInstance()), @@ -63,17 +65,46 @@ public static OpenTelemetry get(TelemetrySettings telemetrySettings, Settings se * @param spanExporter span exporter instance * @param contextPropagators context propagator instance * @param sampler sampler instance - * @return Opentelemetry instance + * @return OpenTelemetrySdk instance */ - public static OpenTelemetry get(Settings settings, SpanExporter spanExporter, ContextPropagators contextPropagators, Sampler sampler) { + public static OpenTelemetrySdk get( + Settings settings, + SpanExporter spanExporter, + ContextPropagators contextPropagators, + Sampler sampler + ) { Resource resource = Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "OpenSearch")); - SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() + SdkTracerProvider sdkTracerProvider = createSdkTracerProvider(settings, spanExporter, sampler, resource); + SdkMeterProvider sdkMeterProvider = createSdkMetricProvider(settings, resource); + return OpenTelemetrySdk.builder() + .setTracerProvider(sdkTracerProvider) + .setMeterProvider(sdkMeterProvider) + .setPropagators(contextPropagators) + .buildAndRegisterGlobal(); + } + + private static SdkMeterProvider createSdkMetricProvider(Settings settings, Resource resource) { + return SdkMeterProvider.builder() + .setResource(resource) + .registerMetricReader( + PeriodicMetricReader.builder(OTelMetricsExporterFactory.create(settings)) + .setInterval(TelemetrySettings.METRICS_PUBLISH_INTERVAL_SETTING.get(settings).getSeconds(), TimeUnit.SECONDS) + .build() + ) + .build(); + } + + private static SdkTracerProvider createSdkTracerProvider( + Settings settings, + SpanExporter spanExporter, + Sampler sampler, + Resource resource + ) { + return SdkTracerProvider.builder() .addSpanProcessor(spanProcessor(settings, spanExporter)) .setResource(resource) .setSampler(sampler) .build(); - - return OpenTelemetrySdk.builder().setTracerProvider(sdkTracerProvider).setPropagators(contextPropagators).buildAndRegisterGlobal(); } private static BatchSpanProcessor spanProcessor(Settings settings, SpanExporter spanExporter) { diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java index 53066ad4ad444..f88afe623fd56 100644 --- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java +++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java @@ -8,41 +8,38 @@ package org.opensearch.telemetry.tracing; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.opensearch.telemetry.OTelAttributesConverter; +import org.opensearch.telemetry.OTelTelemetryPlugin; import java.io.Closeable; import java.io.IOException; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; /** * OTel based Telemetry provider */ -public class OTelTracingTelemetry implements TracingTelemetry { - - private static final Logger logger = LogManager.getLogger(OTelTracingTelemetry.class); +public class OTelTracingTelemetry implements TracingTelemetry { private final OpenTelemetry openTelemetry; + private final T tracerProvider; private final io.opentelemetry.api.trace.Tracer otelTracer; /** - * Creates OTel based Telemetry + * Creates OTel based {@link TracingTelemetry} * @param openTelemetry OpenTelemetry instance + * @param tracerProvider {@link TracerProvider} instance. */ - public OTelTracingTelemetry(OpenTelemetry openTelemetry) { + public OTelTracingTelemetry(OpenTelemetry openTelemetry, T tracerProvider) { this.openTelemetry = openTelemetry; - this.otelTracer = openTelemetry.getTracer("os-tracer"); - + this.tracerProvider = tracerProvider; + this.otelTracer = tracerProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME); } @Override - public void close() { - try { - ((Closeable) openTelemetry).close(); - } catch (IOException e) { - logger.warn("Error while closing Opentelemetry", e); - } + public void close() throws IOException { + tracerProvider.close(); } @Override diff --git a/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy index 726db3d3f4700..9d529ed5a2a56 100644 --- a/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/telemetry-otel/src/main/plugin-metadata/plugin-security.policy @@ -11,6 +11,7 @@ grant { permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.net.NetPermission "getProxySelector"; permission java.net.SocketPermission "*", "connect,resolve"; + permission java.util.PropertyPermission "*", "read,write"; }; diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java index 8c2b5d14733e2..2fcf89947e537 100644 --- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/OTelTelemetryPluginTests.java @@ -12,12 +12,15 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.telemetry.metrics.MetricsTelemetry; +import org.opensearch.telemetry.metrics.OTelMetricsTelemetry; import org.opensearch.telemetry.tracing.OTelTracingTelemetry; import org.opensearch.telemetry.tracing.TracingTelemetry; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; +import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -25,6 +28,7 @@ import java.util.Set; import static org.opensearch.telemetry.OTelTelemetryPlugin.OTEL_TRACER_NAME; +import static org.opensearch.telemetry.OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING; import static org.opensearch.telemetry.OTelTelemetrySettings.OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING; import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_BATCH_SIZE_SETTING; import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING; @@ -34,41 +38,47 @@ public class OTelTelemetryPluginTests extends OpenSearchTestCase { - private OTelTelemetryPlugin oTelTracerModulePlugin; + private OTelTelemetryPlugin oTelTelemetryPlugin; private Optional telemetry; private TracingTelemetry tracingTelemetry; + private MetricsTelemetry metricsTelemetry; + @Before public void setup() { // TRACER_EXPORTER_DELAY_SETTING should always be less than 10 seconds because // io.opentelemetry.sdk.OpenTelemetrySdk.close waits only for 10 seconds for shutdown to complete. Settings settings = Settings.builder().put(TRACER_EXPORTER_DELAY_SETTING.getKey(), "1s").build(); - oTelTracerModulePlugin = new OTelTelemetryPlugin(settings); - telemetry = oTelTracerModulePlugin.getTelemetry( + oTelTelemetryPlugin = new OTelTelemetryPlugin(settings); + telemetry = oTelTelemetryPlugin.getTelemetry( new TelemetrySettings(Settings.EMPTY, new ClusterSettings(settings, Set.of(TRACER_ENABLED_SETTING, TRACER_SAMPLER_PROBABILITY))) ); tracingTelemetry = telemetry.get().getTracingTelemetry(); + metricsTelemetry = telemetry.get().getMetricsTelemetry(); } public void testGetTelemetry() { Set> allTracerSettings = new HashSet<>(); ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add)); - assertEquals(OTEL_TRACER_NAME, oTelTracerModulePlugin.getName()); + assertEquals(OTEL_TRACER_NAME, oTelTelemetryPlugin.getName()); assertTrue(tracingTelemetry instanceof OTelTracingTelemetry); + assertTrue(metricsTelemetry instanceof OTelMetricsTelemetry); assertEquals( Arrays.asList( TRACER_EXPORTER_BATCH_SIZE_SETTING, TRACER_EXPORTER_DELAY_SETTING, TRACER_EXPORTER_MAX_QUEUE_SIZE_SETTING, - OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING + OTEL_TRACER_SPAN_EXPORTER_CLASS_SETTING, + OTEL_METRICS_EXPORTER_CLASS_SETTING ), - oTelTracerModulePlugin.getSettings() + oTelTelemetryPlugin.getSettings() ); } @After - public void cleanup() { + public void cleanup() throws IOException { tracingTelemetry.close(); + metricsTelemetry.close(); } } diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java new file mode 100644 index 0000000000000..233c93e6b9a36 --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.telemetry.OTelAttributesConverter; +import org.opensearch.telemetry.OTelTelemetryPlugin; +import org.opensearch.telemetry.metrics.tags.Tags; +import org.opensearch.test.OpenSearchTestCase; + +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.api.metrics.DoubleCounterBuilder; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.api.metrics.DoubleUpDownCounterBuilder; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.LongUpDownCounterBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class OTelMetricsTelemetryTests extends OpenSearchTestCase { + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void testCounter() { + String counterName = "test-counter"; + String description = "test"; + String unit = "1"; + Meter mockMeter = mock(Meter.class); + DoubleCounter mockOTelDoubleCounter = mock(DoubleCounter.class); + LongCounterBuilder mockOTelLongCounterBuilder = mock(LongCounterBuilder.class); + DoubleCounterBuilder mockOTelDoubleCounterBuilder = mock(DoubleCounterBuilder.class); + MeterProvider meterProvider = mock(MeterProvider.class); + when(meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockMeter); + MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(meterProvider); + when(mockMeter.counterBuilder(counterName)).thenReturn(mockOTelLongCounterBuilder); + when(mockOTelLongCounterBuilder.setDescription(description)).thenReturn(mockOTelLongCounterBuilder); + when(mockOTelLongCounterBuilder.setUnit(unit)).thenReturn(mockOTelLongCounterBuilder); + when(mockOTelLongCounterBuilder.ofDoubles()).thenReturn(mockOTelDoubleCounterBuilder); + when(mockOTelDoubleCounterBuilder.build()).thenReturn(mockOTelDoubleCounter); + + Counter counter = metricsTelemetry.createCounter(counterName, description, unit); + counter.add(1.0); + verify(mockOTelDoubleCounter).add(1.0); + Tags tags = Tags.create().addTag("test", "test"); + counter.add(2.0, tags); + verify(mockOTelDoubleCounter).add(2.0, OTelAttributesConverter.convert(tags)); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void testCounterNegativeValue() { + String counterName = "test-counter"; + String description = "test"; + String unit = "1"; + Meter mockMeter = mock(Meter.class); + DoubleCounter mockOTelDoubleCounter = mock(DoubleCounter.class); + LongCounterBuilder mockOTelLongCounterBuilder = mock(LongCounterBuilder.class); + DoubleCounterBuilder mockOTelDoubleCounterBuilder = mock(DoubleCounterBuilder.class); + + MeterProvider meterProvider = mock(MeterProvider.class); + when(meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockMeter); + MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(meterProvider); + when(mockMeter.counterBuilder(counterName)).thenReturn(mockOTelLongCounterBuilder); + when(mockOTelLongCounterBuilder.setDescription(description)).thenReturn(mockOTelLongCounterBuilder); + when(mockOTelLongCounterBuilder.setUnit(unit)).thenReturn(mockOTelLongCounterBuilder); + when(mockOTelLongCounterBuilder.ofDoubles()).thenReturn(mockOTelDoubleCounterBuilder); + when(mockOTelDoubleCounterBuilder.build()).thenReturn(mockOTelDoubleCounter); + + Counter counter = metricsTelemetry.createCounter(counterName, description, unit); + counter.add(-1.0); + verify(mockOTelDoubleCounter).add(-1.0); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void testUpDownCounter() { + String counterName = "test-counter"; + String description = "test"; + String unit = "1"; + Meter mockMeter = mock(Meter.class); + DoubleUpDownCounter mockOTelUpDownDoubleCounter = mock(DoubleUpDownCounter.class); + LongUpDownCounterBuilder mockOTelLongUpDownCounterBuilder = mock(LongUpDownCounterBuilder.class); + DoubleUpDownCounterBuilder mockOTelDoubleUpDownCounterBuilder = mock(DoubleUpDownCounterBuilder.class); + + MeterProvider meterProvider = mock(MeterProvider.class); + when(meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockMeter); + MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(meterProvider); + when(mockMeter.upDownCounterBuilder(counterName)).thenReturn(mockOTelLongUpDownCounterBuilder); + when(mockOTelLongUpDownCounterBuilder.setDescription(description)).thenReturn(mockOTelLongUpDownCounterBuilder); + when(mockOTelLongUpDownCounterBuilder.setUnit(unit)).thenReturn(mockOTelLongUpDownCounterBuilder); + when(mockOTelLongUpDownCounterBuilder.ofDoubles()).thenReturn(mockOTelDoubleUpDownCounterBuilder); + when(mockOTelDoubleUpDownCounterBuilder.build()).thenReturn(mockOTelUpDownDoubleCounter); + + Counter counter = metricsTelemetry.createUpDownCounter(counterName, description, unit); + counter.add(1.0); + verify(mockOTelUpDownDoubleCounter).add(1.0); + Tags tags = Tags.create().addTag("test", "test"); + counter.add(-2.0, tags); + verify(mockOTelUpDownDoubleCounter).add((-2.0), OTelAttributesConverter.convert(tags)); + } +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/DummyMetricExporter.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/DummyMetricExporter.java new file mode 100644 index 0000000000000..65c52911dbef9 --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/DummyMetricExporter.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics.exporter; + +import java.util.Collection; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.MetricExporter; + +public class DummyMetricExporter implements MetricExporter { + @Override + public CompletableResultCode export(Collection metrics) { + return null; + } + + @Override + public CompletableResultCode flush() { + return null; + } + + @Override + public CompletableResultCode shutdown() { + return null; + } + + @Override + public AggregationTemporality getAggregationTemporality(InstrumentType instrumentType) { + return null; + } +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactoryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactoryTests.java new file mode 100644 index 0000000000000..e68da030bfb52 --- /dev/null +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/exporter/OTelMetricsExporterFactoryTests.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics.exporter; + +import org.opensearch.common.settings.Settings; +import org.opensearch.telemetry.OTelTelemetrySettings; +import org.opensearch.test.OpenSearchTestCase; + +import io.opentelemetry.exporter.logging.LoggingMetricExporter; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.sdk.metrics.export.MetricExporter; + +public class OTelMetricsExporterFactoryTests extends OpenSearchTestCase { + + public void testMetricsExporterDefault() { + Settings settings = Settings.builder().build(); + MetricExporter metricExporter = OTelMetricsExporterFactory.create(settings); + assertTrue(metricExporter instanceof LoggingMetricExporter); + } + + public void testMetricsExporterLogging() { + Settings settings = Settings.builder() + .put( + OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.getKey(), + "io.opentelemetry.exporter.logging.LoggingMetricExporter" + ) + .build(); + MetricExporter metricExporter = OTelMetricsExporterFactory.create(settings); + assertTrue(metricExporter instanceof LoggingMetricExporter); + } + + public void testMetricExporterInvalid() { + Settings settings = Settings.builder().put(OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.getKey(), "abc").build(); + assertThrows(IllegalArgumentException.class, () -> OTelMetricsExporterFactory.create(settings)); + } + + public void testMetricExporterNoCreateFactoryMethod() { + Settings settings = Settings.builder() + .put( + OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.getKey(), + "org.opensearch.telemetry.metrics.exporter.DummyMetricExporter" + ) + .build(); + IllegalStateException exception = assertThrows(IllegalStateException.class, () -> OTelMetricsExporterFactory.create(settings)); + assertEquals( + "MetricExporter instantiation failed for class [org.opensearch.telemetry.metrics.exporter.DummyMetricExporter]", + exception.getMessage() + ); + } + + public void testMetricExporterNonMetricExporterClass() { + Settings settings = Settings.builder() + .put(OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.getKey(), "java.lang.String") + .build(); + IllegalStateException exception = assertThrows(IllegalStateException.class, () -> OTelMetricsExporterFactory.create(settings)); + assertEquals("MetricExporter instantiation failed for class [java.lang.String]", exception.getMessage()); + assertTrue(exception.getCause() instanceof NoSuchMethodError); + + } + + public void testMetricExporterGetDefaultMethod() { + Settings settings = Settings.builder() + .put( + OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.getKey(), + "io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter" + ) + .build(); + + assertTrue(OTelMetricsExporterFactory.create(settings) instanceof OtlpGrpcMetricExporter); + } + +} diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelAttributesConverterTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelAttributesConverterTests.java index d992daec1b7bb..ee67384d01759 100644 --- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelAttributesConverterTests.java +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelAttributesConverterTests.java @@ -8,6 +8,8 @@ package org.opensearch.telemetry.tracing; +import org.opensearch.telemetry.OTelAttributesConverter; +import org.opensearch.telemetry.metrics.tags.Tags; import org.opensearch.telemetry.tracing.attributes.Attributes; import org.opensearch.test.OpenSearchTestCase; @@ -19,13 +21,13 @@ public class OTelAttributesConverterTests extends OpenSearchTestCase { public void testConverterNullAttributes() { - io.opentelemetry.api.common.Attributes otelAttributes = OTelAttributesConverter.convert(null); + io.opentelemetry.api.common.Attributes otelAttributes = OTelAttributesConverter.convert((Attributes) null); assertEquals(0, otelAttributes.size()); } public void testConverterEmptyAttributes() { Attributes attributes = Attributes.EMPTY; - io.opentelemetry.api.common.Attributes otelAttributes = OTelAttributesConverter.convert(null); + io.opentelemetry.api.common.Attributes otelAttributes = OTelAttributesConverter.convert(attributes); assertEquals(0, otelAttributes.size()); } @@ -47,4 +49,12 @@ public void testConverterMultipleAttributes() { assertEquals(4, otelAttributes.size()); otelAttributes.asMap().forEach((x, y) -> assertEquals(attributeMap.get(x.getKey()), y)); } + + public void testConverterMultipleTags() { + Tags tags = Tags.create().addTag("key1", 1l).addTag("key2", 1.0).addTag("key3", true).addTag("key4", "value4"); + Map tagsMap = tags.getTagsMap(); + io.opentelemetry.api.common.Attributes otelAttributes = OTelAttributesConverter.convert(tags); + assertEquals(4, otelAttributes.size()); + otelAttributes.asMap().forEach((x, y) -> assertEquals(tagsMap.get(x.getKey()), y)); + } } diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java index 505756318ff62..1a508ed252493 100644 --- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java +++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java @@ -8,16 +8,15 @@ package org.opensearch.telemetry.tracing; +import org.opensearch.telemetry.OTelTelemetryPlugin; import org.opensearch.telemetry.tracing.attributes.Attributes; import org.opensearch.test.OpenSearchTestCase; -import java.util.Collections; -import java.util.Map; - import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -26,29 +25,31 @@ import static org.mockito.Mockito.when; public class OTelTracingTelemetryTests extends OpenSearchTestCase { - + @SuppressWarnings({ "rawtypes", "unchecked" }) public void testCreateSpanWithoutParent() { OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); Tracer mockTracer = mock(Tracer.class); - when(mockOpenTelemetry.getTracer("os-tracer")).thenReturn(mockTracer); + TracerProvider mockTracerProvider = mock(TracerProvider.class); + when(mockTracerProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockTracer); SpanBuilder mockSpanBuilder = mock(SpanBuilder.class); when(mockTracer.spanBuilder("span_name")).thenReturn(mockSpanBuilder); when(mockSpanBuilder.setAllAttributes(any(io.opentelemetry.api.common.Attributes.class))).thenReturn(mockSpanBuilder); when(mockSpanBuilder.startSpan()).thenReturn(mock(io.opentelemetry.api.trace.Span.class)); when(mockSpanBuilder.setSpanKind(any(io.opentelemetry.api.trace.SpanKind.class))).thenReturn(mockSpanBuilder); - Map attributeMap = Collections.singletonMap("name", "value"); Attributes attributes = Attributes.create().addAttribute("name", "value"); - TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); + TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider); Span span = tracingTelemetry.createSpan(SpanCreationContext.internal().name("span_name").attributes(attributes), null); verify(mockSpanBuilder, never()).setParent(any()); verify(mockSpanBuilder).setAllAttributes(createAttribute(attributes)); assertNull(span.getParentSpan()); } + @SuppressWarnings({ "rawtypes", "unchecked" }) public void testCreateSpanWithParent() { OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); Tracer mockTracer = mock(Tracer.class); - when(mockOpenTelemetry.getTracer("os-tracer")).thenReturn(mockTracer); + TracerProvider mockTracerProvider = mock(TracerProvider.class); + when(mockTracerProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockTracer); SpanBuilder mockSpanBuilder = mock(SpanBuilder.class); when(mockTracer.spanBuilder("span_name")).thenReturn(mockSpanBuilder); when(mockSpanBuilder.setParent(any())).thenReturn(mockSpanBuilder); @@ -58,7 +59,7 @@ public void testCreateSpanWithParent() { Span parentSpan = new OTelSpan("parent_span", mock(io.opentelemetry.api.trace.Span.class), null); - TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); + TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider); Attributes attributes = Attributes.create().addAttribute("name", 1l); Span span = tracingTelemetry.createSpan(SpanCreationContext.internal().name("span_name").attributes(attributes), parentSpan); @@ -69,10 +70,12 @@ public void testCreateSpanWithParent() { assertEquals("parent_span", span.getParentSpan().getSpanName()); } + @SuppressWarnings({ "rawtypes", "unchecked" }) public void testCreateSpanWithParentWithMultipleAttributes() { OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); Tracer mockTracer = mock(Tracer.class); - when(mockOpenTelemetry.getTracer("os-tracer")).thenReturn(mockTracer); + TracerProvider mockTracerProvider = mock(TracerProvider.class); + when(mockTracerProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockTracer); SpanBuilder mockSpanBuilder = mock(SpanBuilder.class); when(mockTracer.spanBuilder("span_name")).thenReturn(mockSpanBuilder); when(mockSpanBuilder.setParent(any())).thenReturn(mockSpanBuilder); @@ -82,7 +85,7 @@ public void testCreateSpanWithParentWithMultipleAttributes() { Span parentSpan = new OTelSpan("parent_span", mock(io.opentelemetry.api.trace.Span.class), null); - TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); + TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider); Attributes attributes = Attributes.create() .addAttribute("key1", 1l) .addAttribute("key2", 2.0) @@ -115,12 +118,14 @@ private io.opentelemetry.api.common.Attributes createAttributeLong(Attributes at return attributesBuilder.build(); } + @SuppressWarnings({ "rawtypes", "unchecked" }) public void testGetContextPropagator() { OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class); Tracer mockTracer = mock(Tracer.class); - when(mockOpenTelemetry.getTracer("os-tracer")).thenReturn(mockTracer); + TracerProvider mockTracerProvider = mock(TracerProvider.class); + when(mockTracerProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockTracer); - TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry); + TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider); assertTrue(tracingTelemetry.getContextPropagator() instanceof OTelTracingContextPropagator); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 02f6bdd5ad24c..f44c8c8bea7f1 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -224,6 +224,9 @@ import org.opensearch.tasks.consumer.TopNSearchTasksLogger; import org.opensearch.telemetry.TelemetryModule; import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.MetricsRegistryFactory; +import org.opensearch.telemetry.metrics.NoopMetricsRegistryFactory; import org.opensearch.telemetry.tracing.NoopTracerFactory; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.telemetry.tracing.TracerFactory; @@ -392,6 +395,8 @@ public static class DiscoverySettings { private final LocalNodeFactory localNodeFactory; private final NodeService nodeService; private final Tracer tracer; + + private final MetricsRegistry metricsRegistry; final NamedWriteableRegistry namedWriteableRegistry; private final AtomicReference runnableTaskListener; private FileCache fileCache; @@ -591,17 +596,22 @@ protected Node( } TracerFactory tracerFactory; + MetricsRegistryFactory metricsRegistryFactory; if (FeatureFlags.isEnabled(TELEMETRY)) { final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings()); List telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class); TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, telemetrySettings); tracerFactory = new TracerFactory(telemetrySettings, telemetryModule.getTelemetry(), threadPool.getThreadContext()); + metricsRegistryFactory = new MetricsRegistryFactory(telemetrySettings, telemetryModule.getTelemetry()); } else { tracerFactory = new NoopTracerFactory(); + metricsRegistryFactory = new NoopMetricsRegistryFactory(); } tracer = tracerFactory.getTracer(); + metricsRegistry = metricsRegistryFactory.getMetricsRegistry(); resourcesToClose.add(tracer::close); + resourcesToClose.add(metricsRegistry::close); final IngestService ingestService = new IngestService( clusterService, threadPool, @@ -1206,6 +1216,7 @@ protected Node( b.bind(IdentityService.class).toInstance(identityService); b.bind(Tracer.class).toInstance(tracer); b.bind(SearchRequestStats.class).toInstance(searchRequestStats); + b.bind(MetricsRegistry.class).toInstance(metricsRegistry); b.bind(RemoteClusterStateService.class).toProvider(() -> remoteClusterStateService); b.bind(PersistedStateRegistry.class).toInstance(persistedStateRegistry); }); @@ -1573,6 +1584,7 @@ public synchronized void close() throws IOException { toClose.add(stopWatch::stop); if (FeatureFlags.isEnabled(TELEMETRY)) { toClose.add(injector.getInstance(Tracer.class)); + toClose.add(injector.getInstance(MetricsRegistry.class)); } if (logger.isTraceEnabled()) { diff --git a/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java b/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java index dc0b04244296f..edb20cfa9dfc5 100644 --- a/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java +++ b/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java @@ -12,6 +12,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; /** * Wrapper class to encapsulate tracing related settings @@ -39,6 +40,16 @@ public class TelemetrySettings { Setting.Property.Dynamic ); + /** + * metrics publish interval in seconds. + */ + public static final Setting METRICS_PUBLISH_INTERVAL_SETTING = Setting.timeSetting( + "telemetry.otel.metrics.publish.interval", + TimeValue.timeValueSeconds(60), + Setting.Property.NodeScope, + Setting.Property.Final + ); + private volatile boolean tracingEnabled; private volatile double samplingProbability; diff --git a/server/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistryFactory.java b/server/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistryFactory.java new file mode 100644 index 0000000000000..c7e2229c18437 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/metrics/MetricsRegistryFactory.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.annotation.InternalApi; +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; +import org.opensearch.telemetry.tracing.Tracer; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Optional; + +/** + * {@link MetricsRegistryFactory} represents a single global class that is used to access {@link MetricsRegistry}s. + *

+ * The {@link MetricsRegistry} singleton object can be retrieved using MetricsRegistryFactory::getMetricsRegistry. The {@link MetricsRegistryFactory} object + * is created during class initialization and cannot subsequently be changed. + * + * @opensearch.internal + */ +@InternalApi +public class MetricsRegistryFactory implements Closeable { + + private static final Logger logger = LogManager.getLogger(MetricsRegistryFactory.class); + + private final TelemetrySettings telemetrySettings; + private final MetricsRegistry metricsRegistry; + + public MetricsRegistryFactory(TelemetrySettings telemetrySettings, Optional telemetry) { + this.telemetrySettings = telemetrySettings; + this.metricsRegistry = metricsRegistry(telemetry); + } + + /** + * Returns the {@link MetricsRegistry} instance + * + * @return MetricsRegistry instance + */ + public MetricsRegistry getMetricsRegistry() { + return metricsRegistry; + } + + /** + * Closes the {@link Tracer} + */ + @Override + public void close() { + try { + metricsRegistry.close(); + } catch (IOException e) { + logger.warn("Error closing MetricsRegistry", e); + } + } + + private MetricsRegistry metricsRegistry(Optional telemetry) { + MetricsRegistry metricsRegistry = telemetry.map(Telemetry::getMetricsTelemetry) + .map(metricsTelemetry -> createDefaultMetricsRegistry(metricsTelemetry)) + .orElse(NoopMetricsRegistry.INSTANCE); + return metricsRegistry; + } + + private MetricsRegistry createDefaultMetricsRegistry(MetricsTelemetry metricsTelemetry) { + return new DefaultMetricsRegistry(metricsTelemetry); + } + +} diff --git a/server/src/main/java/org/opensearch/telemetry/metrics/NoopMetricsRegistryFactory.java b/server/src/main/java/org/opensearch/telemetry/metrics/NoopMetricsRegistryFactory.java new file mode 100644 index 0000000000000..5137cb18e2cc0 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/metrics/NoopMetricsRegistryFactory.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.common.annotation.InternalApi; +import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; + +import java.util.Optional; + +/** + * No-op implementation of {@link MetricsRegistryFactory} + * + * @opensearch.internal + */ +@InternalApi +public class NoopMetricsRegistryFactory extends MetricsRegistryFactory { + public NoopMetricsRegistryFactory() { + super(null, Optional.empty()); + } + + @Override + public MetricsRegistry getMetricsRegistry() { + return NoopMetricsRegistry.INSTANCE; + } + + @Override + public void close() { + + } +} diff --git a/server/src/main/java/org/opensearch/telemetry/metrics/package-info.java b/server/src/main/java/org/opensearch/telemetry/metrics/package-info.java new file mode 100644 index 0000000000000..ad4564e1d7773 --- /dev/null +++ b/server/src/main/java/org/opensearch/telemetry/metrics/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes needed for telemetry. + */ +package org.opensearch.telemetry.metrics; diff --git a/server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java b/server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java new file mode 100644 index 0000000000000..5d5ea62dd161e --- /dev/null +++ b/server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.telemetry.metrics; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.telemetry.Telemetry; +import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.metrics.noop.NoopCounter; +import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; +import org.opensearch.telemetry.tracing.TracingTelemetry; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MetricsRegistryFactoryTests extends OpenSearchTestCase { + + private MetricsRegistryFactory metricsRegistryFactory; + + @After + public void close() { + metricsRegistryFactory.close(); + } + + public void testGetMeterRegistryWithUnavailableMetricsTelemetry() { + Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), false).build(); + TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); + Telemetry mockTelemetry = mock(Telemetry.class); + when(mockTelemetry.getTracingTelemetry()).thenReturn(mock(TracingTelemetry.class)); + metricsRegistryFactory = new MetricsRegistryFactory(telemetrySettings, Optional.empty()); + + MetricsRegistry metricsRegistry = metricsRegistryFactory.getMetricsRegistry(); + + assertTrue(metricsRegistry instanceof NoopMetricsRegistry); + assertTrue(metricsRegistry.createCounter("test", "test", "test") == NoopCounter.INSTANCE); + assertTrue(metricsRegistry.createUpDownCounter("test", "test", "test") == NoopCounter.INSTANCE); + } + + public void testGetMetricsWithAvailableMetricsTelemetry() { + Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true).build(); + TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings())); + Telemetry mockTelemetry = mock(Telemetry.class); + when(mockTelemetry.getMetricsTelemetry()).thenReturn(mock(MetricsTelemetry.class)); + metricsRegistryFactory = new MetricsRegistryFactory(telemetrySettings, Optional.of(mockTelemetry)); + + MetricsRegistry metricsRegistry = metricsRegistryFactory.getMetricsRegistry(); + assertTrue(metricsRegistry instanceof DefaultMetricsRegistry); + + } + + private Set> getClusterSettings() { + Set> allTracerSettings = new HashSet<>(); + ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add)); + return allTracerSettings; + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java index 6b428a7f65594..95321a7009be9 100644 --- a/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java +++ b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java @@ -10,6 +10,7 @@ import org.opensearch.telemetry.Telemetry; import org.opensearch.telemetry.TelemetrySettings; +import org.opensearch.telemetry.metrics.Counter; import org.opensearch.telemetry.metrics.MetricsTelemetry; import org.opensearch.telemetry.tracing.TracingTelemetry; import org.opensearch.test.telemetry.tracing.MockTracingTelemetry; @@ -34,6 +35,20 @@ public TracingTelemetry getTracingTelemetry() { @Override public MetricsTelemetry getMetricsTelemetry() { return new MetricsTelemetry() { + @Override + public Counter createCounter(String name, String description, String unit) { + return null; + } + + @Override + public Counter createUpDownCounter(String name, String description, String unit) { + return null; + } + + @Override + public void close() { + + } }; } } From 1eec0f103685d2a54f9ee28d59de66a7ec826f11 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Thu, 5 Oct 2023 19:33:04 +0530 Subject: [PATCH 113/388] fix stale remote cluster uuid state not purged from remote (#10016) * fix stale remote cluster uuid state not purged from remote Signed-off-by: bansvaru * fix tests Signed-off-by: bansvaru * use new limit parameter Signed-off-by: bansvaru * minor refactoring Signed-off-by: bansvaru * delete index metadata files before manifest file Signed-off-by: bansvaru * add basic UT Signed-off-by: bansvaru * delete all data related to a cluster uuid in a single call Signed-off-by: bansvaru * fix git diff Signed-off-by: bansvaru * remove unreferenced code Signed-off-by: bansvaru * fix spa Signed-off-by: bansvaru --------- Signed-off-by: bansvaru --- .../remote/RemoteClusterStateService.java | 76 ++++++++++--- .../RemoteClusterStateServiceTests.java | 106 +++++++++++++----- 2 files changed, 137 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index dddc5376803a5..164611816e1dc 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -103,11 +103,11 @@ public class RemoteClusterStateService implements Closeable { Property.Final ); - private static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; - private static final String INDEX_PATH_TOKEN = "index"; - private static final String MANIFEST_PATH_TOKEN = "manifest"; - private static final String MANIFEST_FILE_PREFIX = "manifest"; - private static final String INDEX_METADATA_FILE_PREFIX = "metadata"; + public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; + public static final String INDEX_PATH_TOKEN = "index"; + public static final String MANIFEST_PATH_TOKEN = "manifest"; + public static final String MANIFEST_FILE_PREFIX = "manifest"; + public static final String INDEX_METADATA_FILE_PREFIX = "metadata"; private final String nodeId; private final Supplier repositoriesService; @@ -385,13 +385,20 @@ private void writeIndexMetadataAsync( @Nullable public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) throws IOException { + assert clusterState != null : "Last accepted cluster state is not set"; if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { logger.error("Local node is not elected cluster manager. Exiting"); return null; } - assert clusterState != null : "Last accepted cluster state is not set"; assert previousManifest != null : "Last cluster metadata manifest is not set"; - return uploadManifest(clusterState, previousManifest.getIndices(), previousManifest.getPreviousClusterUUID(), true); + ClusterMetadataManifest committedManifest = uploadManifest( + clusterState, + previousManifest.getIndices(), + previousManifest.getPreviousClusterUUID(), + true + ); + deleteStaleClusterUUIDs(clusterState, committedManifest); + return committedManifest; } @Override @@ -719,30 +726,42 @@ private boolean isInvalidClusterUUID(ClusterMetadataManifest manifest) { } /** - * Fetch latest ClusterMetadataManifest file from remote state store + * Fetch ClusterMetadataManifest files from remote state store in order * * @param clusterUUID uuid of cluster state to refer to in remote * @param clusterName name of the cluster - * @return latest ClusterMetadataManifest filename + * @param limit max no of files to fetch + * @return all manifest file names */ - private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { + private List getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException { try { /** - * {@link BlobContainer#listBlobsByPrefixInSortedOrder} will get the latest manifest file + * {@link BlobContainer#listBlobsByPrefixInSortedOrder} will list the latest manifest file first * as the manifest file name generated via {@link RemoteClusterStateService#getManifestFileName} ensures * when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top. */ - List manifestFilesMetadata = manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder( + return manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder( MANIFEST_FILE_PREFIX + DELIMITER, - 1, + limit, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC ); - if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { - return Optional.of(manifestFilesMetadata.get(0).name()); - } } catch (IOException e) { throw new IllegalStateException("Error while fetching latest manifest file for remote cluster state", e); } + } + + /** + * Fetch latest ClusterMetadataManifest file from remote state store + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @return latest ClusterMetadataManifest filename + */ + private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, 1); + if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { + return Optional.of(manifestFilesMetadata.get(0).name()); + } logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); return Optional.empty(); } @@ -791,7 +810,7 @@ public IndexMetadataTransferException(String errorDesc, Throwable cause) { * @param clusterName name of the cluster * @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged */ - public void deleteStaleClusterMetadata(String clusterName, List clusterUUIDs) { + private void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUIDs) { clusterUUIDs.forEach(clusterUUID -> { getBlobStoreTransferService().deleteAsync( ThreadPool.Names.REMOTE_PURGE, @@ -923,4 +942,27 @@ private void deleteStalePaths(String clusterName, String clusterUUID, List { + String clusterName = clusterState.getClusterName().value(); + logger.info("Deleting stale cluster UUIDs data from remote [{}]", clusterName); + Set allClustersUUIDsInRemote; + try { + allClustersUUIDsInRemote = new HashSet<>(getAllClusterUUIDs(clusterState.getClusterName().value())); + } catch (IOException e) { + logger.info(String.format(Locale.ROOT, "Error while fetching all cluster UUIDs for [%s]", clusterName)); + return; + } + // Retain last 2 cluster uuids data + allClustersUUIDsInRemote.remove(committedManifest.getClusterUUID()); + allClustersUUIDsInRemote.remove(committedManifest.getPreviousClusterUUID()); + deleteStaleUUIDsClusterMetadata(clusterName, new ArrayList<>(allClustersUUIDsInRemote)); + }); + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 65166386733c6..6ecbc23f75bee 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -64,6 +64,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -76,6 +78,8 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class RemoteClusterStateServiceTests extends OpenSearchTestCase { @@ -334,13 +338,8 @@ public void testReadLatestMetadataManifestFailedIOException() throws IOException final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); BlobContainer blobContainer = mockBlobStoreObjects(); - when( - blobContainer.listBlobsByPrefixInSortedOrder( - "manifest" + RemoteClusterStateService.DELIMITER, - 1, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ) - ).thenThrow(IOException.class); + when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) + .thenThrow(IOException.class); remoteClusterStateService.start(); Exception e = assertThrows( @@ -357,13 +356,8 @@ public void testReadLatestMetadataManifestFailedNoManifestFileInRemote() throws final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); BlobContainer blobContainer = mockBlobStoreObjects(); - when( - blobContainer.listBlobsByPrefixInSortedOrder( - "manifest" + RemoteClusterStateService.DELIMITER, - 1, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ) - ).thenReturn(List.of()); + when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) + .thenReturn(List.of()); remoteClusterStateService.start(); Optional manifest = remoteClusterStateService.getLatestClusterMetadataManifest( @@ -378,13 +372,8 @@ public void testReadLatestMetadataManifestFailedManifestFileRemoveAfterFetchInRe BlobContainer blobContainer = mockBlobStoreObjects(); BlobMetadata blobMetadata = new PlainBlobMetadata("manifestFileName", 1); - when( - blobContainer.listBlobsByPrefixInSortedOrder( - "manifest" + RemoteClusterStateService.DELIMITER, - 1, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ) - ).thenReturn(Arrays.asList(blobMetadata)); + when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) + .thenReturn(Arrays.asList(blobMetadata)); when(blobContainer.readBlob("manifestFileName")).thenThrow(FileNotFoundException.class); remoteClusterStateService.start(); @@ -618,6 +607,72 @@ public void testGetValidPreviousClusterUUIDWithInvalidMultipleChains() throws IO assertThrows(IllegalStateException.class, () -> remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")); } + public void testDeleteStaleClusterUUIDs() throws IOException { + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + ClusterMetadataManifest clusterMetadataManifest = ClusterMetadataManifest.builder() + .indices(List.of()) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID(randomAlphaOfLength(10)) + .clusterUUID("cluster-uuid1") + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID(ClusterState.UNKNOWN_UUID) + .committed(true) + .build(); + + BlobPath blobPath = new BlobPath().add("random-path"); + when((blobStoreRepository.basePath())).thenReturn(blobPath); + BlobContainer uuidContainerContainer = mock(BlobContainer.class); + BlobContainer manifest2Container = mock(BlobContainer.class); + BlobContainer manifest3Container = mock(BlobContainer.class); + when(blobStore.blobContainer(any())).then(invocation -> { + BlobPath blobPath1 = invocation.getArgument(0); + if (blobPath1.buildAsString().endsWith("cluster-state/")) { + return uuidContainerContainer; + } else if (blobPath1.buildAsString().contains("cluster-state/cluster-uuid2/")) { + return manifest2Container; + } else if (blobPath1.buildAsString().contains("cluster-state/cluster-uuid3/")) { + return manifest3Container; + } else { + throw new IllegalArgumentException("Unexpected blob path " + blobPath1); + } + }); + Map blobMetadataMap = Map.of( + "cluster-uuid1", + mock(BlobContainer.class), + "cluster-uuid2", + mock(BlobContainer.class), + "cluster-uuid3", + mock(BlobContainer.class) + ); + when(uuidContainerContainer.children()).thenReturn(blobMetadataMap); + when( + manifest2Container.listBlobsByPrefixInSortedOrder( + MANIFEST_FILE_PREFIX + DELIMITER, + Integer.MAX_VALUE, + BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC + ) + ).thenReturn(List.of(new PlainBlobMetadata("mainfest2", 1L))); + when( + manifest3Container.listBlobsByPrefixInSortedOrder( + MANIFEST_FILE_PREFIX + DELIMITER, + Integer.MAX_VALUE, + BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC + ) + ).thenReturn(List.of(new PlainBlobMetadata("mainfest3", 1L))); + remoteClusterStateService.start(); + remoteClusterStateService.deleteStaleClusterUUIDs(clusterState, clusterMetadataManifest); + try { + assertBusy(() -> { + verify(manifest2Container, times(1)).delete(); + verify(manifest3Container, times(1)).delete(); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { final BlobPath blobPath = mock(BlobPath.class); when((blobStoreRepository.basePath())).thenReturn(blobPath); @@ -760,13 +815,8 @@ private void mockBlobContainer( Map indexMetadataMap ) throws IOException { BlobMetadata blobMetadata = new PlainBlobMetadata("manifestFileName", 1); - when( - blobContainer.listBlobsByPrefixInSortedOrder( - "manifest" + RemoteClusterStateService.DELIMITER, - 1, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ) - ).thenReturn(Arrays.asList(blobMetadata)); + when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) + .thenReturn(Arrays.asList(blobMetadata)); BytesReference bytes = RemoteClusterStateService.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( clusterMetadataManifest, From c6c07f3afd5658cb797ccbbf1a4f5dceb4f67015 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:52:42 -0400 Subject: [PATCH 114/388] Bump org.bouncycastle:bc-fips from 1.0.2.3 to 1.0.2.4 in /distribution/tools/plugin-cli (#10297) * Bump org.bouncycastle:bc-fips in /distribution/tools/plugin-cli Bumps org.bouncycastle:bc-fips from 1.0.2.3 to 1.0.2.4. --- updated-dependencies: - dependency-name: org.bouncycastle:bc-fips dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + distribution/tools/plugin-cli/build.gradle | 2 +- distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.3.jar.sha1 | 1 - distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.3.jar.sha1 create mode 100644 distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 153cc12c5f9bc..1d82f15198878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303)) - Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301)) - Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210)) +- Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 2db3fef55d02e..b61a00aba04bc 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -38,7 +38,7 @@ dependencies { compileOnly project(":server") compileOnly project(":libs:opensearch-cli") api "org.bouncycastle:bcpg-fips:1.0.7.1" - api "org.bouncycastle:bc-fips:1.0.2.3" + api "org.bouncycastle:bc-fips:1.0.2.4" testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.3.0' testRuntimeOnly("com.google.guava:guava:${versions.guava}") { diff --git a/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.3.jar.sha1 b/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.3.jar.sha1 deleted file mode 100644 index c71320050b7de..0000000000000 --- a/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da62b32cb72591f5b4d322e6ab0ce7de3247b534 \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1 b/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1 new file mode 100644 index 0000000000000..da37449f80d7e --- /dev/null +++ b/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1 @@ -0,0 +1 @@ +9008d04fc13da6455e6a792935b93b629757335d \ No newline at end of file From 12ccc33e192aab203d17c68e979c54b90b818da1 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 5 Oct 2023 23:38:12 +0530 Subject: [PATCH 115/388] Update TODO for updating public documentation (#10408) Signed-off-by: Bhumika Saini --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index cd7c1bb980eec..c47b9e0b69256 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -19,7 +19,7 @@ Resolves #[Issue number to be closed when this PR is merged] - [ ] New functionality has javadoc added - [ ] Commits are signed per the DCO using --signoff - [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog)) -- [ ] GitHub issue/PR created in [OpenSearch documentation repo](https://github.com/opensearch-project/documentation-website) for the required public documentation changes (#[Issue/PR number]) +- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose) By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). From 0facd50bc72a7ca05f44a167a9794c270cd0ad83 Mon Sep 17 00:00:00 2001 From: Prabhat <20185657+CaptainDredge@users.noreply.github.com> Date: Thu, 5 Oct 2023 12:56:07 -0700 Subject: [PATCH 116/388] Race condition fix for datetime optimization (#10385) * Race condition fix for datetime optimization Signed-off-by: Prabhat Sharma * Changed JavaDateTimeFormatter caching of parser from MRU(most recently used) to a simple last used formatter Signed-off-by: Prabhat Sharma --------- Signed-off-by: Prabhat Sharma Co-authored-by: Prabhat Sharma --- .../common/time/JavaDateFormatter.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java b/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java index 594ebc3b26a73..14191357fb8aa 100644 --- a/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java +++ b/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java @@ -74,6 +74,7 @@ class JavaDateFormatter implements DateFormatter { private final List parsers; private final JavaDateFormatter roundupParser; private final Boolean canCacheLastParsedFormatter; + private volatile DateTimeFormatter lastParsedformatter = null; /** * A round up formatter @@ -150,7 +151,7 @@ JavaDateFormatter getRoundupParser() { if (parsers.length == 0) { this.parsers = Collections.singletonList(printer); } else { - this.parsers = new CopyOnWriteArrayList<>(parsers); + this.parsers = Arrays.asList(parsers); } List roundUp = createRoundUpParser(format, roundupParserConsumer); this.roundupParser = new RoundUpFormatter(format, roundUp); @@ -235,7 +236,7 @@ private JavaDateFormatter( this.printFormat = printFormat; this.printer = printer; this.roundupParser = roundUpParsers != null ? new RoundUpFormatter(format, roundUpParsers) : null; - this.parsers = new CopyOnWriteArrayList<>(parsers); + this.parsers = parsers; this.canCacheLastParsedFormatter = canCacheLastParsedFormatter; } @@ -286,24 +287,22 @@ public TemporalAccessor parse(String input) { private TemporalAccessor doParse(String input) { if (parsers.size() > 1) { Object object = null; - DateTimeFormatter lastParsedformatter = null; + if (canCacheLastParsedFormatter && lastParsedformatter != null) { + ParsePosition pos = new ParsePosition(0); + object = lastParsedformatter.toFormat().parseObject(input, pos); + if (parsingSucceeded(object, input, pos)) { + return (TemporalAccessor) object; + } + } for (DateTimeFormatter formatter : parsers) { ParsePosition pos = new ParsePosition(0); object = formatter.toFormat().parseObject(input, pos); if (parsingSucceeded(object, input, pos)) { lastParsedformatter = formatter; - break; + return (TemporalAccessor) object; } } - if (lastParsedformatter != null) { - if (canCacheLastParsedFormatter && lastParsedformatter != parsers.get(0)) { - synchronized (parsers) { - parsers.remove(lastParsedformatter); - parsers.add(0, lastParsedformatter); - } - } - return (TemporalAccessor) object; - } + throw new DateTimeParseException("Failed to parse with all enclosed parsers", input, 0); } return this.parsers.get(0).parse(input); From fdaa438ee03db285417bc0bd4204a0fbd4e699c9 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Fri, 6 Oct 2023 04:22:42 +0800 Subject: [PATCH 117/388] Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API (#10101) * Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API Signed-off-by: Gao Binlong * modify change log Signed-off-by: Gao Binlong * Add more tests Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong Signed-off-by: Daniel (dB.) Doubrovkine Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 1 + .../rest-api-spec/test/ingest/90_simulate.yml | 137 ++++++++++++++++++ .../ingest/SimulatePipelineRequest.java | 28 +++- .../SimulatePipelineRequestParsingTests.java | 60 +++++++- 4 files changed, 215 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d82f15198878..f9d2123458335 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -143,6 +143,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045)) - Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082)) - Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089)) +- Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101)) - Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194)) - Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256)) diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml index e012a82b15927..7c073739f6a1f 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml @@ -976,3 +976,140 @@ teardown: } - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.root_cause.0.reason: "Pipeline processor configured for non-existent pipeline [____pipeline_doesnot_exist___]" } + +--- +"Test simulate with docs containing metadata fields": + - do: + ingest.simulate: + body: > + { + "pipeline": { + "description": "_description", + "processors": [ + { + "set" : { + "field": "field2", + "value": "foo" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_id": "id", + "_routing": "foo", + "_version": 100, + "_if_seq_no": 12333333333333333, + "_if_primary_term": 1, + "_source": { + "foo": "bar" + } + } + ] + } + + - length: { docs: 1 } + - match: { docs.0.doc._index: "index" } + - match: { docs.0.doc._id: "id" } + - match: { docs.0.doc._routing: "foo" } + - match: { docs.0.doc._version: "100" } + - match: { docs.0.doc._if_seq_no: "12333333333333333" } + - match: { docs.0.doc._if_primary_term: "1" } + - match: { docs.0.doc._source.foo: "bar" } + + - do: + catch: bad_request + ingest.simulate: + body: > + { + "pipeline": { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field2", + "value": "foo" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_id": "id", + "_routing": "foo", + "_version": "bar", + "_source": { + "foo": "bar" + } + } + ] + } + - match: { status: 400 } + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "Failed to parse parameter [_version], only int or long is accepted" } + + - do: + catch: bad_request + ingest.simulate: + body: > + { + "pipeline": { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field2", + "value": "foo" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_id": "id", + "_routing": "foo", + "_if_seq_no": "123", + "_source": { + "foo": "bar" + } + } + ] + } + - match: { status: 400 } + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "Failed to parse parameter [_if_seq_no], only int or long is accepted" } + + - do: + catch: bad_request + ingest.simulate: + body: > + { + "pipeline": { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field2", + "value": "foo" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_id": "id", + "_routing": "foo", + "_if_primary_term": "1", + "_source": { + "foo": "bar" + } + } + ] + } + - match: { status: 400 } + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "Failed to parse parameter [_if_primary_term], only int or long is accepted" } diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java index 2234934499609..ec3ee981b646f 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java @@ -218,7 +218,12 @@ private static List parseDocs(Map config) { String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, dataMap, Metadata.ROUTING.getFieldName()); Long version = null; if (dataMap.containsKey(Metadata.VERSION.getFieldName())) { - version = (Long) ConfigurationUtils.readObject(null, null, dataMap, Metadata.VERSION.getFieldName()); + Object versionFieldValue = ConfigurationUtils.readObject(null, null, dataMap, Metadata.VERSION.getFieldName()); + if (versionFieldValue instanceof Integer || versionFieldValue instanceof Long) { + version = ((Number) versionFieldValue).longValue(); + } else { + throw new IllegalArgumentException("Failed to parse parameter [_version], only int or long is accepted"); + } } VersionType versionType = null; if (dataMap.containsKey(Metadata.VERSION_TYPE.getFieldName())) { @@ -228,12 +233,25 @@ private static List parseDocs(Map config) { } IngestDocument ingestDocument = new IngestDocument(index, id, routing, version, versionType, document); if (dataMap.containsKey(Metadata.IF_SEQ_NO.getFieldName())) { - Long ifSeqNo = (Long) ConfigurationUtils.readObject(null, null, dataMap, Metadata.IF_SEQ_NO.getFieldName()); - ingestDocument.setFieldValue(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo); + Object ifSeqNoFieldValue = ConfigurationUtils.readObject(null, null, dataMap, Metadata.IF_SEQ_NO.getFieldName()); + if (ifSeqNoFieldValue instanceof Integer || ifSeqNoFieldValue instanceof Long) { + ingestDocument.setFieldValue(Metadata.IF_SEQ_NO.getFieldName(), ((Number) ifSeqNoFieldValue).longValue()); + } else { + throw new IllegalArgumentException("Failed to parse parameter [_if_seq_no], only int or long is accepted"); + } } if (dataMap.containsKey(Metadata.IF_PRIMARY_TERM.getFieldName())) { - Long ifPrimaryTerm = (Long) ConfigurationUtils.readObject(null, null, dataMap, Metadata.IF_PRIMARY_TERM.getFieldName()); - ingestDocument.setFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm); + Object ifPrimaryTermFieldValue = ConfigurationUtils.readObject( + null, + null, + dataMap, + Metadata.IF_PRIMARY_TERM.getFieldName() + ); + if (ifPrimaryTermFieldValue instanceof Integer || ifPrimaryTermFieldValue instanceof Long) { + ingestDocument.setFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), ((Number) ifPrimaryTermFieldValue).longValue()); + } else { + throw new IllegalArgumentException("Failed to parse parameter [_if_primary_term], only int or long is accepted"); + } } ingestDocumentList.add(ingestDocument); } diff --git a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java index 705fb546a2fed..04a9d08bb22bc 100644 --- a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -144,17 +144,29 @@ public void innerTestParseWithProvidedPipeline() throws Exception { List fields = Arrays.asList(INDEX, ID, ROUTING, VERSION, VERSION_TYPE, IF_SEQ_NO, IF_PRIMARY_TERM); for (IngestDocument.Metadata field : fields) { if (field == VERSION) { - Long value = randomLong(); - doc.put(field.getFieldName(), value); - expectedDoc.put(field.getFieldName(), value); + if (randomBoolean()) { + Long value = randomLong(); + doc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), value); + } else { + Integer value = randomIntBetween(1, 1000000); + doc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), value); + } } else if (field == VERSION_TYPE) { String value = VersionType.toString(randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL, VersionType.EXTERNAL_GTE)); doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); } else if (field == IF_SEQ_NO || field == IF_PRIMARY_TERM) { - Long value = randomNonNegativeLong(); - doc.put(field.getFieldName(), value); - expectedDoc.put(field.getFieldName(), value); + if (randomBoolean()) { + Long value = randomNonNegativeLong(); + doc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), value); + } else { + Integer value = randomIntBetween(1, 1000000); + doc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), value); + } } else { if (randomBoolean()) { String value = randomAlphaOfLengthBetween(1, 10); @@ -282,4 +294,40 @@ public void testNotValidDocs() { ); assertThat(e3.getMessage(), containsString("required property is missing")); } + + public void testNotValidMetadataFields() { + List fields = Arrays.asList(VERSION, IF_SEQ_NO, IF_PRIMARY_TERM); + for (IngestDocument.Metadata field : fields) { + String metadataFieldName = field.getFieldName(); + Map requestContent = new HashMap<>(); + List> docs = new ArrayList<>(); + requestContent.put(Fields.DOCS, docs); + Map doc = new HashMap<>(); + doc.put(metadataFieldName, randomAlphaOfLengthBetween(1, 10)); + doc.put(Fields.SOURCE, Collections.singletonMap(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))); + docs.add(doc); + + Map pipelineConfig = new HashMap<>(); + List> processors = new ArrayList<>(); + Map processorConfig = new HashMap<>(); + List> onFailureProcessors = new ArrayList<>(); + int numOnFailureProcessors = randomIntBetween(0, 1); + for (int j = 0; j < numOnFailureProcessors; j++) { + onFailureProcessors.add(Collections.singletonMap("mock_processor", Collections.emptyMap())); + } + if (numOnFailureProcessors > 0) { + processorConfig.put("on_failure", onFailureProcessors); + } + processors.add(Collections.singletonMap("mock_processor", processorConfig)); + pipelineConfig.put("processors", processors); + + requestContent.put(Fields.PIPELINE, pipelineConfig); + + assertThrows( + "Failed to parse parameter [" + metadataFieldName + "], only int or long is accepted", + IllegalArgumentException.class, + () -> SimulatePipelineRequest.parse(requestContent, false, ingestService) + ); + } + } } From 651a9aa2ca05244e74bcaabcc89d878977efe259 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Thu, 5 Oct 2023 13:40:14 -0700 Subject: [PATCH 118/388] Segment Replication - Fix ShardLockObtained error during corruption cases (#10370) * Segment Replication - Fix ShardLockObtained error during corruption cases This change fixes a bug where shards could not be recreated locally after corruption. This occured because the store was not decref'd to 0 if the commit on close would fail with a corruption exception. Signed-off-by: Marc Handalian * Remove exra logs Signed-off-by: Marc Handalian * Remove flaky assertion on store refcount Signed-off-by: Marc Handalian * Remove flaky test. Signed-off-by: Marc Handalian * PR Feedback. Remove hacky handling of corruption when fetching metadata. This will now check for store corruption when replication has failed and fail the shard accordingly. This commit also fixes logging in NRTReplicationEngine. Signed-off-by: Marc Handalian * Fix unit test. Signed-off-by: Marc Handalian * Fix test failure testSegRepSucceedsOnPreviousCopiedFiles. This test broke because we invoked target.indexShard on a closed replicationTarget. In these cases we can assume the store is not corrupt. Signed-off-by: Marc Handalian * spotless Signed-off-by: Marc Handalian * Revert flaky IT Signed-off-by: Marc Handalian * Fix flakiness failure by expecting RTE when check index fails. Signed-off-by: Marc Handalian * reintroduce ITs and use recoveries API instead of waiting on shard state. Signed-off-by: Marc Handalian * Fix edge case where flush failures would not get reported as corruption. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- CHANGELOG.md | 1 + .../replication/SegmentReplicationBaseIT.java | 5 +- .../replication/SegmentReplicationIT.java | 135 ++++++++++++++++++ .../index/engine/NRTReplicationEngine.java | 27 +++- .../org/opensearch/index/store/Store.java | 8 +- .../replication/SegmentReplicationTarget.java | 11 +- .../SegmentReplicationTargetService.java | 26 +++- .../engine/NRTReplicationEngineTests.java | 44 ++++++ .../SegmentReplicationTargetServiceTests.java | 3 +- .../SegmentReplicationTargetTests.java | 3 +- 10 files changed, 245 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d2123458335..5b14f123b41cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,6 +146,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101)) - Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194)) - Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256)) +- Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370)) ### Security diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java index 8e68a8bde39d5..1d93eecd6b245 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java @@ -197,9 +197,10 @@ protected IndexShard getIndexShard(String node, ShardId shardId, String indexNam protected IndexShard getIndexShard(String node, String indexName) { final Index index = resolveIndex(indexName); IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); - IndexService indexService = indicesService.indexServiceSafe(index); + IndexService indexService = indicesService.indexService(index); + assertNotNull(indexService); final Optional shardId = indexService.shardIds().stream().findFirst(); - return indexService.getShard(shardId.get()); + return shardId.map(indexService::getShard).orElse(null); } protected boolean segmentReplicationWithRemoteEnabled() { diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 33bc5a8f3afe6..81556cc270151 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -24,6 +24,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.flush.FlushRequest; +import org.opensearch.action.admin.indices.recovery.RecoveryResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.get.GetResponse; @@ -58,6 +59,7 @@ import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.XContentBuilder; @@ -71,6 +73,7 @@ import org.opensearch.index.engine.NRTReplicationReaderManager; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.recovery.FileChunkRequest; +import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.NodeClosedException; @@ -82,6 +85,7 @@ import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; +import org.opensearch.transport.TransportRequest; import org.opensearch.transport.TransportService; import org.junit.Before; @@ -94,6 +98,7 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static java.util.Arrays.asList; @@ -1777,4 +1782,134 @@ public void testRealtimeTermVectorRequestsUnSuccessful() throws IOException { } + public void testSendCorruptBytesToReplica() throws Exception { + // this test stubs transport calls specific to node-node replication. + assumeFalse( + "Skipping the test as its not compatible with segment replication with remote store.", + segmentReplicationWithRemoteEnabled() + ); + final String primaryNode = internalCluster().startDataOnlyNode(); + createIndex( + INDEX_NAME, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.refresh_interval", -1) + .build() + ); + ensureYellow(INDEX_NAME); + final String replicaNode = internalCluster().startDataOnlyNode(); + ensureGreen(INDEX_NAME); + + MockTransportService primaryTransportService = ((MockTransportService) internalCluster().getInstance( + TransportService.class, + primaryNode + )); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean failed = new AtomicBoolean(false); + primaryTransportService.addSendBehavior( + internalCluster().getInstance(TransportService.class, replicaNode), + (connection, requestId, action, request, options) -> { + if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK) && failed.getAndSet(true) == false) { + FileChunkRequest req = (FileChunkRequest) request; + logger.info("SENDING CORRUPT file chunk [{}] lastChunk: {}", req, req.lastChunk()); + TransportRequest corrupt = new FileChunkRequest( + req.recoveryId(), + ((FileChunkRequest) request).requestSeqNo(), + ((FileChunkRequest) request).shardId(), + ((FileChunkRequest) request).metadata(), + ((FileChunkRequest) request).position(), + new BytesArray("test"), + false, + 0, + 0L + ); + connection.sendRequest(requestId, action, corrupt, options); + latch.countDown(); + } else { + connection.sendRequest(requestId, action, request, options); + } + } + ); + for (int i = 0; i < 100; i++) { + client().prepareIndex(INDEX_NAME) + .setId(String.valueOf(i)) + .setSource(jsonBuilder().startObject().field("field", i).endObject()) + .get(); + } + final long originalRecoveryTime = getRecoveryStopTime(replicaNode); + assertNotEquals(originalRecoveryTime, 0); + refresh(INDEX_NAME); + latch.await(); + assertTrue(failed.get()); + waitForNewPeerRecovery(replicaNode, originalRecoveryTime); + // reset checkIndex to ensure our original shard doesn't throw + resetCheckIndexStatus(); + waitForSearchableDocs(100, primaryNode, replicaNode); + } + + public void testWipeSegmentBetweenSyncs() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + final String primaryNode = internalCluster().startDataOnlyNode(); + createIndex( + INDEX_NAME, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.refresh_interval", -1) + .build() + ); + ensureYellow(INDEX_NAME); + final String replicaNode = internalCluster().startDataOnlyNode(); + ensureGreen(INDEX_NAME); + + for (int i = 0; i < 10; i++) { + client().prepareIndex(INDEX_NAME) + .setId(String.valueOf(i)) + .setSource(jsonBuilder().startObject().field("field", i).endObject()) + .get(); + } + refresh(INDEX_NAME); + ensureGreen(INDEX_NAME); + final long originalRecoveryTime = getRecoveryStopTime(replicaNode); + + final IndexShard indexShard = getIndexShard(replicaNode, INDEX_NAME); + waitForSearchableDocs(INDEX_NAME, 10, List.of(replicaNode)); + indexShard.store().directory().deleteFile("_0.si"); + + for (int i = 11; i < 21; i++) { + client().prepareIndex(INDEX_NAME) + .setId(String.valueOf(i)) + .setSource(jsonBuilder().startObject().field("field", i).endObject()) + .get(); + } + refresh(INDEX_NAME); + waitForNewPeerRecovery(replicaNode, originalRecoveryTime); + resetCheckIndexStatus(); + waitForSearchableDocs(20, primaryNode, replicaNode); + } + + private void waitForNewPeerRecovery(String replicaNode, long originalRecoveryTime) throws Exception { + assertBusy(() -> { + // assert we have a peer recovery after the original + final long time = getRecoveryStopTime(replicaNode); + assertNotEquals(time, 0); + assertNotEquals(originalRecoveryTime, time); + + }, 1, TimeUnit.MINUTES); + } + + private long getRecoveryStopTime(String nodeName) { + final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(INDEX_NAME).get(); + final List recoveryStates = recoveryResponse.shardRecoveryStates().get(INDEX_NAME); + logger.info("Recovery states {}", recoveryResponse); + for (RecoveryState recoveryState : recoveryStates) { + if (recoveryState.getTargetNode().getName().equals(nodeName)) { + return recoveryState.getTimer().stopTime(); + } + } + return 0L; + } } diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index 570a2b186841a..020e92aba4ce5 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -379,6 +379,7 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException { try { commitSegmentInfos(); } catch (IOException e) { + maybeFailEngine("flush", e); throw new FlushFailedEngineException(shardId, e); } finally { flushLock.unlock(); @@ -437,13 +438,29 @@ protected final void closeNoLock(String reason, CountDownLatch closedLatch) { latestSegmentInfos.counter = latestSegmentInfos.counter + SI_COUNTER_INCREMENT; latestSegmentInfos.changed(); } - commitSegmentInfos(latestSegmentInfos); - IOUtils.close(readerManager, translogManager, store::decRef); + try { + commitSegmentInfos(latestSegmentInfos); + } catch (IOException e) { + // mark the store corrupted unless we are closing as result of engine failure. + // in this case Engine#failShard will handle store corruption. + if (failEngineLock.isHeldByCurrentThread() == false && store.isMarkedCorrupted() == false) { + try { + store.markStoreCorrupted(e); + } catch (IOException ex) { + logger.warn("Unable to mark store corrupted", ex); + } + } + } + IOUtils.close(readerManager, translogManager); } catch (Exception e) { - logger.warn("failed to close engine", e); + logger.error("failed to close engine", e); } finally { - logger.debug("engine closed [{}]", reason); - closedLatch.countDown(); + try { + store.decRef(); + logger.debug("engine closed [{}]", reason); + } finally { + closedLatch.countDown(); + } } } } diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 285241ba89996..9f505121d8dfa 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -385,7 +385,13 @@ public MetadataSnapshot getMetadata(SegmentInfos segmentInfos) throws IOExceptio */ public Map getSegmentMetadataMap(SegmentInfos segmentInfos) throws IOException { assert indexSettings.isSegRepEnabled(); - return loadMetadata(segmentInfos, directory, logger, true).fileMetadata; + failIfCorrupted(); + try { + return loadMetadata(segmentInfos, directory, logger, true).fileMetadata; + } catch (NoSuchFileException | CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { + markStoreCorrupted(ex); + throw ex; + } } /** diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 5ae480b7d63a4..0eb6ce36fa63d 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -18,7 +18,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.opensearch.OpenSearchCorruptionException; -import org.opensearch.OpenSearchException; import org.opensearch.action.StepListener; import org.opensearch.common.UUIDs; import org.opensearch.common.lucene.Lucene; @@ -261,9 +260,7 @@ private void finalizeReplication(CheckpointInfoResponse checkpointInfoResponse) } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) { // this is a fatal exception at this stage. // this means we transferred files from the remote that have not be checksummed and they are - // broken. We have to clean up this shard entirely, remove all files and bubble it up to the - // source shard since this index might be broken there as well? The Source can handle this and checks - // its content on disk if possible. + // broken. We have to clean up this shard entirely, remove all files and bubble it up. try { try { store.removeCorruptionMarker(); @@ -279,14 +276,14 @@ private void finalizeReplication(CheckpointInfoResponse checkpointInfoResponse) // In this case the shard is closed at some point while updating the reader. // This can happen when the engine is closed in a separate thread. logger.warn("Shard is already closed, closing replication"); - } catch (OpenSearchException ex) { + } catch (CancellableThreads.ExecutionCancelledException ex) { /* Ignore closed replication target as it can happen due to index shard closed event in a separate thread. In such scenario, ignore the exception */ - assert cancellableThreads.isCancelled() : "Replication target closed but segment replication not cancelled"; + assert cancellableThreads.isCancelled() : "Replication target cancelled but cancellable threads not cancelled"; } catch (Exception ex) { - throw new OpenSearchCorruptionException(ex); + throw new ReplicationFailedException(ex); } finally { if (store != null) { store.decRef(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index ffc4ab86661db..46095adfe96b4 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.index.CorruptIndexException; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchCorruptionException; import org.opensearch.action.support.ChannelActionListener; @@ -28,6 +29,7 @@ import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; +import org.opensearch.index.store.Store; import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.FileChunkRequest; import org.opensearch.indices.recovery.ForceSyncRequest; @@ -46,6 +48,7 @@ import org.opensearch.transport.TransportRequestOptions; import org.opensearch.transport.TransportService; +import java.io.IOException; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; @@ -522,7 +525,7 @@ public void onResponse(Void o) { @Override public void onFailure(Exception e) { logger.debug("Replication failed {}", target.description()); - if (e instanceof OpenSearchCorruptionException) { + if (isStoreCorrupt(target) || e instanceof CorruptIndexException || e instanceof OpenSearchCorruptionException) { onGoingReplications.fail(replicationId, new ReplicationFailedException("Store corruption during replication", e), true); return; } @@ -531,6 +534,27 @@ public void onFailure(Exception e) { }); } + private boolean isStoreCorrupt(SegmentReplicationTarget target) { + // ensure target is not already closed. In that case + // we can assume the store is not corrupt and that the replication + // event completed successfully. + if (target.refCount() > 0) { + final Store store = target.store(); + if (store.tryIncRef()) { + try { + return store.isMarkedCorrupted(); + } catch (IOException ex) { + logger.warn("Unable to determine if store is corrupt", ex); + return false; + } finally { + store.decRef(); + } + } + } + // store already closed. + return false; + } + private class FileChunkTransportRequestHandler implements TransportRequestHandler { // How many bytes we've copied since we last called RateLimiter.pause diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java index 7b587e9a83d2d..ee25d3789fb13 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java @@ -577,6 +577,50 @@ public void testDecrefToZeroRemovesFile() throws IOException { } } + public void testCommitOnCloseThrowsException_decRefStore() throws Exception { + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + + final Store nrtEngineStore = createStore(INDEX_SETTINGS, newDirectory()); + final NRTReplicationEngine nrtEngine = buildNrtReplicaEngine(globalCheckpoint, nrtEngineStore, INDEX_SETTINGS); + List operations = generateHistoryOnReplica( + randomIntBetween(1, 10), + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + indexOperations(nrtEngine, operations); + // wipe the nrt directory initially so we can sync with primary. + cleanAndCopySegmentsFromPrimary(nrtEngine); + nrtEngineStore.directory().deleteFile("_0.si"); + assertEquals(2, nrtEngineStore.refCount()); + nrtEngine.close(); + assertEquals(1, nrtEngineStore.refCount()); + assertTrue(nrtEngineStore.isMarkedCorrupted()); + // store will throw when eventually closed, not handled here. + assertThrows(RuntimeException.class, nrtEngineStore::close); + } + + public void testFlushThrowsFlushFailedExceptionOnCorruption() throws Exception { + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + + final Store nrtEngineStore = createStore(INDEX_SETTINGS, newDirectory()); + final NRTReplicationEngine nrtEngine = buildNrtReplicaEngine(globalCheckpoint, nrtEngineStore, INDEX_SETTINGS); + List operations = generateHistoryOnReplica( + randomIntBetween(1, 10), + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + indexOperations(nrtEngine, operations); + // wipe the nrt directory initially so we can sync with primary. + cleanAndCopySegmentsFromPrimary(nrtEngine); + nrtEngineStore.directory().deleteFile("_0.si"); + assertThrows(FlushFailedEngineException.class, nrtEngine::flush); + assertTrue(nrtEngineStore.isMarkedCorrupted()); + // store will throw when eventually closed, not handled here. + assertThrows(RuntimeException.class, nrtEngineStore::close); + } + private void copySegments(Collection latestPrimaryFiles, Engine nrtEngine) throws IOException { final Store store = nrtEngine.store; final List replicaFiles = List.of(store.directory().listAll()); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index c1f88a6938d33..c108de5ee5ea6 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -9,6 +9,7 @@ package org.opensearch.indices.replication; import org.apache.lucene.store.AlreadyClosedException; +import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.cluster.ClusterState; @@ -553,7 +554,7 @@ public void testForceSegmentSyncHandlerWithFailure() throws Exception { ).txGet(); }); Throwable nestedException = finalizeException.getCause().getCause(); - assertTrue(nestedException instanceof IOException); + assertNotNull(ExceptionsHelper.unwrap(finalizeException, IOException.class)); assertTrue(nestedException.getMessage().contains("dummy failure")); } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index 2596dd6e62026..a9d7d3cdd32fc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -249,7 +249,7 @@ public void onFailure(Exception e) { }); } - public void testFailure_finalizeReplication_IOException() throws IOException { + public void testFailure_finalizeReplication_NonCorruptionException() throws IOException { IOException exception = new IOException("dummy failure"); SegmentReplicationSource segrepSource = new TestReplicationSource() { @@ -288,6 +288,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { + assertEquals(ReplicationFailedException.class, e.getClass()); assertEquals(exception, e.getCause()); segrepTarget.fail(new ReplicationFailedException(e), false); } From 10bae207cc6d8fa6bbb42021ef967c0bb57787a5 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 5 Oct 2023 13:41:15 -0700 Subject: [PATCH 119/388] Fix error handling for future completion (#10406) Signed-off-by: Kunal Kotwani --- .../repositories/s3/S3BlobContainer.java | 34 +++++++------- .../s3/S3BlobStoreContainerTests.java | 45 +++++++++++++++++++ 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index fcfccf50ad326..9ffdba5eaae3a 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -241,23 +241,27 @@ public void readBlobAsync(String blobName, ActionListener listener) return; } - final List blobPartInputStreamFutures = new ArrayList<>(); - final long blobSize = blobMetadata.objectSize(); - final Integer numberOfParts = blobMetadata.objectParts() == null ? null : blobMetadata.objectParts().totalPartsCount(); - final String blobChecksum = blobMetadata.checksum().checksumCRC32(); - - if (numberOfParts == null) { - blobPartInputStreamFutures.add(() -> getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, null)); - } else { - // S3 multipart files use 1 to n indexing - for (int partNumber = 1; partNumber <= numberOfParts; partNumber++) { - final int innerPartNumber = partNumber; - blobPartInputStreamFutures.add( - () -> getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, innerPartNumber) - ); + try { + final List blobPartInputStreamFutures = new ArrayList<>(); + final long blobSize = blobMetadata.objectSize(); + final Integer numberOfParts = blobMetadata.objectParts() == null ? null : blobMetadata.objectParts().totalPartsCount(); + final String blobChecksum = blobMetadata.checksum() == null ? null : blobMetadata.checksum().checksumCRC32(); + + if (numberOfParts == null) { + blobPartInputStreamFutures.add(() -> getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, null)); + } else { + // S3 multipart files use 1 to n indexing + for (int partNumber = 1; partNumber <= numberOfParts; partNumber++) { + final int innerPartNumber = partNumber; + blobPartInputStreamFutures.add( + () -> getBlobPartInputStreamContainer(s3AsyncClient, bucketName, blobKey, innerPartNumber) + ); + } } + listener.onResponse(new ReadContext(blobSize, blobPartInputStreamFutures, blobChecksum)); + } catch (Exception ex) { + listener.onFailure(ex); } - listener.onResponse(new ReadContext(blobSize, blobPartInputStreamFutures, blobChecksum)); }); } catch (Exception ex) { listener.onFailure(SdkException.create("Error occurred while fetching blob parts from the repository", ex)); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index 2e54705e9cd78..e266bba372d80 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -1074,6 +1074,51 @@ public void testReadBlobAsyncFailure() throws Exception { assertEquals(1, readContextActionListener.getFailureCount()); } + public void testReadBlobAsyncOnCompleteFailureMissingData() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final String blobName = randomAlphaOfLengthBetween(1, 10); + final String checksum = randomAlphaOfLength(10); + + final long objectSize = 100L; + final int objectPartCount = 10; + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference( + AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null) + ); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + final BlobPath blobPath = new BlobPath(); + + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.serverSideEncryption()).thenReturn(false); + when(blobStore.asyncClientReference()).thenReturn(amazonAsyncS3Reference); + + CompletableFuture getObjectAttributesResponseCompletableFuture = new CompletableFuture<>(); + getObjectAttributesResponseCompletableFuture.complete( + GetObjectAttributesResponse.builder() + .checksum(Checksum.builder().build()) + .objectSize(null) + .objectParts(GetObjectAttributesParts.builder().totalPartsCount(objectPartCount).build()) + .build() + ); + when(s3AsyncClient.getObjectAttributes(any(GetObjectAttributesRequest.class))).thenReturn( + getObjectAttributesResponseCompletableFuture + ); + + CountDownLatch countDownLatch = new CountDownLatch(1); + CountingCompletionListener readContextActionListener = new CountingCompletionListener<>(); + LatchedActionListener listener = new LatchedActionListener<>(readContextActionListener, countDownLatch); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + blobContainer.readBlobAsync(blobName, listener); + countDownLatch.await(); + + assertEquals(0, readContextActionListener.getResponseCount()); + assertEquals(1, readContextActionListener.getFailureCount()); + } + public void testGetBlobMetadata() throws Exception { final String checksum = randomAlphaOfLengthBetween(1, 10); final long objectSize = 100L; From a3f8432df30c69be3950ca55984032a1b2323f35 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:56:09 -0700 Subject: [PATCH 120/388] [Remote Store] Fix stats reporting for multistream downloads. (#10402) * Fix stats reporting for multistream downloads. Signed-off-by: Rishikesh1159 * rename tracker to fileTransferTracker. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 --- .../opensearch/index/shard/IndexShard.java | 4 ++- .../store/RemoteSegmentStoreDirectory.java | 35 +++++++++++++++---- .../RemoteStoreReplicationSource.java | 7 ++-- .../RemoteSegmentStoreDirectoryTests.java | 13 ++++--- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 4f08411c19b55..833c91c1766c8 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -160,6 +160,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; @@ -4929,9 +4930,10 @@ private void downloadSegments( final Runnable onFileSync ) throws IOException { final Path indexPath = store.shardPath() == null ? null : store.shardPath().resolveIndex(); + final DirectoryFileTransferTracker fileTransferTracker = store.getDirectoryFileTransferTracker(); for (String segment : toDownloadSegments) { final PlainActionFuture segmentListener = PlainActionFuture.newFuture(); - sourceRemoteDirectory.copyTo(segment, storeDirectory, indexPath, segmentListener); + sourceRemoteDirectory.copyTo(segment, storeDirectory, indexPath, fileTransferTracker, segmentListener); segmentListener.actionGet(); onFileSync.run(); if (targetRemoteDirectory != null) { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index a97b22360716c..a067cb9c5ae61 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -487,17 +487,40 @@ public void copyFrom(Directory from, String src, IOContext context, ActionListen * @param source The source file name * @param destinationDirectory The destination directory (if multipart is not supported) * @param destinationPath The destination path (if multipart is supported) + * @param fileTransferTracker Tracker used for file transfer stats * @param fileCompletionListener The listener to notify of completion */ - public void copyTo(String source, Directory destinationDirectory, Path destinationPath, ActionListener fileCompletionListener) { + public void copyTo( + String source, + Directory destinationDirectory, + Path destinationPath, + DirectoryFileTransferTracker fileTransferTracker, + ActionListener fileCompletionListener + ) { final String blobName = getExistingRemoteFilename(source); if (destinationPath != null && remoteDataDirectory.getBlobContainer() instanceof AsyncMultiStreamBlobContainer) { + long length = 0L; + try { + length = fileLength(source); + } catch (IOException ex) { + logger.error("Unable to fetch segment length for stats tracking", ex); + } + final long fileLength = length; + final long startTime = System.currentTimeMillis(); + fileTransferTracker.addTransferredBytesStarted(fileLength); final AsyncMultiStreamBlobContainer blobContainer = (AsyncMultiStreamBlobContainer) remoteDataDirectory.getBlobContainer(); final Path destinationFilePath = destinationPath.resolve(source); + final ActionListener completionListener = ActionListener.wrap(response -> { + fileTransferTracker.addTransferredBytesSucceeded(fileLength, startTime); + fileCompletionListener.onResponse(response); + }, e -> { + fileTransferTracker.addTransferredBytesFailed(fileLength, startTime); + fileCompletionListener.onFailure(e); + }); final ReadContextListener readContextListener = new ReadContextListener( blobName, destinationFilePath, - fileCompletionListener, + completionListener, threadPool, remoteDataDirectory.getDownloadRateLimiter(), recoverySettings.getMaxConcurrentRemoteStoreStreams() @@ -505,12 +528,10 @@ public void copyTo(String source, Directory destinationDirectory, Path destinati blobContainer.readBlobAsync(blobName, readContextListener); } else { // Fallback to older mechanism of downloading the file - try { + ActionListener.completeWith(fileCompletionListener, () -> { destinationDirectory.copyFrom(this, source, source, IOContext.DEFAULT); - fileCompletionListener.onResponse(source); - } catch (IOException e) { - fileCompletionListener.onFailure(e); - } + return source; + }); } } diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java index e17c5293c38ac..ddbcb86269aa9 100644 --- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java @@ -20,6 +20,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; @@ -121,7 +122,8 @@ public void getSegmentFiles( assert directoryFiles.contains(file) == false : "Local store already contains the file " + file; toDownloadSegments.add(fileMetadata); } - downloadSegments(storeDirectory, remoteDirectory, toDownloadSegments, shardPath, listener); + final DirectoryFileTransferTracker fileTransferTracker = indexShard.store().getDirectoryFileTransferTracker(); + downloadSegments(storeDirectory, remoteDirectory, toDownloadSegments, shardPath, fileTransferTracker, listener); logger.debug("Downloaded segment files from remote store {}", toDownloadSegments); } finally { indexShard.store().decRef(); @@ -138,12 +140,13 @@ private void downloadSegments( RemoteSegmentStoreDirectory remoteStoreDirectory, List toDownloadSegments, ShardPath shardPath, + DirectoryFileTransferTracker fileTransferTracker, ActionListener completionListener ) { final Path indexPath = shardPath == null ? null : shardPath.resolveIndex(); for (StoreFileMetadata storeFileMetadata : toDownloadSegments) { final PlainActionFuture segmentListener = PlainActionFuture.newFuture(); - remoteStoreDirectory.copyTo(storeFileMetadata.name(), storeDirectory, indexPath, segmentListener); + remoteStoreDirectory.copyTo(storeFileMetadata.name(), storeDirectory, indexPath, fileTransferTracker, segmentListener); segmentListener.actionGet(); } completionListener.onResponse(new GetSegmentFilesResponse(toDownloadSegments)); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index b574ccaac55e1..4a89b3c718f0b 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -596,10 +596,15 @@ public void onResponse(String unused) { public void onFailure(Exception e) {} }; Path path = createTempDir(); - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, completionListener); + DirectoryFileTransferTracker directoryFileTransferTracker = new DirectoryFileTransferTracker(); + long sourceFileLengthInBytes = remoteSegmentStoreDirectory.fileLength(filename); + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, directoryFileTransferTracker, completionListener); assertTrue(downloadLatch.await(5000, TimeUnit.SECONDS)); verify(blobContainer, times(1)).readBlobAsync(contains(filename), any()); verify(storeDirectory, times(0)).copyFrom(any(), any(), any(), any()); + + // Verify stats are updated to DirectoryFileTransferTracker + assertEquals(sourceFileLengthInBytes, directoryFileTransferTracker.getTransferredBytesSucceeded()); } public void testCopyFilesTo() throws Exception { @@ -619,7 +624,7 @@ public void onResponse(String unused) { public void onFailure(Exception e) {} }; Path path = createTempDir(); - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, completionListener); + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, new DirectoryFileTransferTracker(), completionListener); assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); } @@ -643,7 +648,7 @@ public void onResponse(String unused) { @Override public void onFailure(Exception e) {} }; - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, null, completionListener); + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, null, new DirectoryFileTransferTracker(), completionListener); assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); } @@ -670,7 +675,7 @@ public void onFailure(Exception e) { } }; Path path = createTempDir(); - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, completionListener); + remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, new DirectoryFileTransferTracker(), completionListener); assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); } From 66aef13cf8bdd0f4d058bc0cd9205415349d4b81 Mon Sep 17 00:00:00 2001 From: Harish Bhakuni Date: Thu, 5 Oct 2023 14:08:54 -0700 Subject: [PATCH 121/388] Updating the separator for RemoteStoreLockManager since underscore is allowed in base64UUID url charset (#10379) * Refactor Remote Store Metadata Lock Manager Utils Signed-off-by: Harish Bhakuni * Address PR Comments Signed-off-by: Harish Bhakuni * Address PR Comments Signed-off-by: Harish Bhakuni * Update Changelog entry Signed-off-by: Harish Bhakuni * Update Changelog entry Signed-off-by: Harish Bhakuni * Unmute testDeleteShallowCopySnapshot test Signed-off-by: Harish Bhakuni --------- Signed-off-by: Harish Bhakuni Co-authored-by: Harish Bhakuni --- CHANGELOG.md | 1 + .../snapshots/DeleteSnapshotIT.java | 1 - .../index/store/lockmanager/FileLockInfo.java | 32 ++++++++++---- .../RemoteStoreLockManagerUtils.java | 7 +++- .../store/lockmanager/FileLockInfoTests.java | 38 ++++++++++++++++- .../BlobStoreRepositoryHelperTests.java | 4 +- .../BlobStoreRepositoryRemoteIndexTests.java | 42 ++++++++++--------- 7 files changed, 91 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b14f123b41cc..4ea8d5ae4cfe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356)) - [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569)) - [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241)) +- Updating the separator for RemoteStoreLockManager since underscore is allowed in base64UUID url charset ([#10379](https://github.com/opensearch-project/OpenSearch/pull/10379)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 31abc16bba50e..e79bf1c16b586 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -64,7 +64,6 @@ public void testDeleteSnapshot() throws Exception { assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9115") public void testDeleteShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java b/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java index 24f42743e1a04..b6be60c489a6c 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/FileLockInfo.java @@ -21,6 +21,7 @@ public class FileLockInfo implements LockInfo { private String fileToLock; private String acquirerId; + private static final int INVALID_INDEX = -1; public String getAcquirerId() { return acquirerId; @@ -88,21 +89,34 @@ static String generateLockName(String fileToLock, String acquirerId) { } public static String getFileToLockNameFromLock(String lockName) { - String[] lockNameTokens = lockName.split(RemoteStoreLockManagerUtils.SEPARATOR); - - if (lockNameTokens.length != 2) { - throw new IllegalArgumentException("Provided Lock Name " + lockName + " is not Valid."); + // use proper separator for the lock file depending on the version it is created + String lockSeparator = lockName.endsWith(RemoteStoreLockManagerUtils.PRE_OS210_LOCK_FILE_EXTENSION) + ? RemoteStoreLockManagerUtils.PRE_OS210_LOCK_SEPARATOR + : RemoteStoreLockManagerUtils.SEPARATOR; + final int indexOfSeparator = lockName.lastIndexOf(lockSeparator); + if (indexOfSeparator == INVALID_INDEX) { + throw new IllegalArgumentException("Provided lock name: " + lockName + " is invalid with separator: " + lockSeparator); } - return lockNameTokens[0]; + return lockName.substring(0, indexOfSeparator); } public static String getAcquirerIdFromLock(String lockName) { - String[] lockNameTokens = lockName.split(RemoteStoreLockManagerUtils.SEPARATOR); + String lockExtension = RemoteStoreLockManagerUtils.LOCK_FILE_EXTENSION; + String lockSeparator = RemoteStoreLockManagerUtils.SEPARATOR; - if (lockNameTokens.length != 2) { - throw new IllegalArgumentException("Provided Lock Name " + lockName + " is not Valid."); + // check if lock file is created on version <=2.10 + if (lockName.endsWith(RemoteStoreLockManagerUtils.PRE_OS210_LOCK_FILE_EXTENSION)) { + lockSeparator = RemoteStoreLockManagerUtils.PRE_OS210_LOCK_SEPARATOR; + lockExtension = RemoteStoreLockManagerUtils.PRE_OS210_LOCK_FILE_EXTENSION; + } + final int indexOfSeparator = lockName.lastIndexOf(lockSeparator); + final int indexOfExt = lockName.lastIndexOf(lockExtension); + if (indexOfSeparator == INVALID_INDEX || indexOfExt == INVALID_INDEX) { + throw new IllegalArgumentException( + "Provided lock name: " + lockName + " is invalid with separator: " + lockSeparator + " and extension: " + lockExtension + ); } - return lockNameTokens[1].replace(RemoteStoreLockManagerUtils.LOCK_FILE_EXTENSION, ""); + return lockName.substring(indexOfSeparator + lockSeparator.length(), indexOfExt); } } diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerUtils.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerUtils.java index 452dfc329d88b..d5fb2722a64dc 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerUtils.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerUtils.java @@ -15,8 +15,11 @@ */ public class RemoteStoreLockManagerUtils { static final String FILE_TO_LOCK_NAME = "file_to_lock"; - static final String SEPARATOR = "___"; - static final String LOCK_FILE_EXTENSION = ".lock"; + static final String PRE_OS210_LOCK_SEPARATOR = "___"; + static final String SEPARATOR = "..."; + // for versions <= 2.10, we have lock files with this extension. + static final String PRE_OS210_LOCK_FILE_EXTENSION = ".lock"; + static final String LOCK_FILE_EXTENSION = ".v2_lock"; static final String ACQUIRER_ID = "acquirer_id"; public static final String NO_TTL = "-1"; static final String LOCK_EXPIRY_TIME = "lock_expiry_time"; diff --git a/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java b/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java index f3a2f1859923e..80413d4cb6612 100644 --- a/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java +++ b/server/src/test/java/org/opensearch/index/store/lockmanager/FileLockInfoTests.java @@ -15,10 +15,24 @@ public class FileLockInfoTests extends OpenSearchTestCase { String testMetadata = "testMetadata"; String testAcquirerId = "testAcquirerId"; + String testAcquirerId2 = "ZxZ4Wh89SXyEPmSYAHrIrQ"; + String testAcquirerId3 = "ZxZ4Wh89SXyEPmSYAHrItS"; + String testMetadata1 = "metadata__9223372036854775806__9223372036854775803__9223372036854775790" + + "__9223372036854775800___Hf3Dbw2QQagfGLlVBOUrg__9223370340398865071__1"; + + String oldLock = testMetadata1 + RemoteStoreLockManagerUtils.PRE_OS210_LOCK_SEPARATOR + testAcquirerId2 + + RemoteStoreLockManagerUtils.PRE_OS210_LOCK_FILE_EXTENSION; + String newLock = testMetadata1 + RemoteStoreLockManagerUtils.SEPARATOR + testAcquirerId3 + + RemoteStoreLockManagerUtils.LOCK_FILE_EXTENSION; public void testGenerateLockName() { FileLockInfo fileLockInfo = FileLockInfo.getLockInfoBuilder().withFileToLock(testMetadata).withAcquirerId(testAcquirerId).build(); assertEquals(fileLockInfo.generateLockName(), FileLockInfo.LockFileUtils.generateLockName(testMetadata, testAcquirerId)); + + // validate that lock generated will be the new version lock + fileLockInfo = FileLockInfo.getLockInfoBuilder().withFileToLock(testMetadata1).withAcquirerId(testAcquirerId3).build(); + assertEquals(fileLockInfo.generateLockName(), newLock); + } public void testGenerateLockNameFailureCase1() { @@ -41,13 +55,33 @@ public void testGetLockPrefixFailureCase() { assertThrows(IllegalArgumentException.class, fileLockInfo::getLockPrefix); } + public void testGetFileToLockNameFromLock() { + assertEquals(testMetadata1, FileLockInfo.LockFileUtils.getFileToLockNameFromLock(oldLock)); + assertEquals(testMetadata1, FileLockInfo.LockFileUtils.getFileToLockNameFromLock(newLock)); + } + + public void testGetAcquirerIdFromLock() { + assertEquals(testAcquirerId2, FileLockInfo.LockFileUtils.getAcquirerIdFromLock(oldLock)); + assertEquals(testAcquirerId3, FileLockInfo.LockFileUtils.getAcquirerIdFromLock(newLock)); + } + public void testGetLocksForAcquirer() throws NoSuchFileException { + String[] locks = new String[] { FileLockInfo.LockFileUtils.generateLockName(testMetadata, testAcquirerId), - FileLockInfo.LockFileUtils.generateLockName(testMetadata, "acquirerId2") }; + FileLockInfo.LockFileUtils.generateLockName(testMetadata, "acquirerId2"), + oldLock, + newLock }; FileLockInfo fileLockInfo = FileLockInfo.getLockInfoBuilder().withAcquirerId(testAcquirerId).build(); - assertEquals(fileLockInfo.getLockForAcquirer(locks), FileLockInfo.LockFileUtils.generateLockName(testMetadata, testAcquirerId)); + + // validate old lock + fileLockInfo = FileLockInfo.getLockInfoBuilder().withAcquirerId(testAcquirerId2).build(); + assertEquals(fileLockInfo.getLockForAcquirer(locks), oldLock); + + // validate new lock + fileLockInfo = FileLockInfo.getLockInfoBuilder().withAcquirerId(testAcquirerId3).build(); + assertEquals(fileLockInfo.getLockForAcquirer(locks), newLock); } } diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java index 4e60cddd14d73..57c126b85ff70 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java @@ -66,7 +66,9 @@ protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String rem BlobPath shardLevelBlobPath = remoteStorerepository.basePath().add(indexUUID).add("0").add("segments").add("lock_files"); BlobContainer blobContainer = remoteStorerepository.blobStore().blobContainer(shardLevelBlobPath); try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) { - return Arrays.stream(lockDirectory.listAll()).filter(lock -> lock.endsWith(".lock")).toArray(String[]::new); + return Arrays.stream(lockDirectory.listAll()) + .filter(lock -> lock.endsWith(".lock") || lock.endsWith(".v2_lock")) + .toArray(String[]::new); } } diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index e3e1bf31e82dc..9cca495cced72 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -145,7 +145,7 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { final SnapshotId snapshotId1 = snapshotInfo.snapshotId(); String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 0) : "there should be no lock files present in directory, but found " + Arrays.toString(lockFiles); + assertEquals("there should be no lock files present in directory, but found " + Arrays.toString(lockFiles), 0, lockFiles.length); logger.info("--> create remote index shallow snapshot"); Settings snapshotRepoSettingsForShallowCopy = Settings.builder() .put(snapshotRepoSettings) @@ -161,8 +161,8 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { final SnapshotId snapshotId2 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId2.getUUID() + ".lock"); + assertEquals("there should be only one lock file, but found " + Arrays.toString(lockFiles), 1, lockFiles.length); + assertTrue(lockFiles[0].endsWith(snapshotId2.getUUID() + ".v2_lock")); logger.info("--> create another normal snapshot"); updateRepository(client, snapshotRepositoryName, snapshotRepoSettings); @@ -174,8 +174,8 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { final SnapshotId snapshotId3 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId2.getUUID() + ".lock"); + assertEquals("there should be only one lock file, but found " + Arrays.toString(lockFiles), 1, lockFiles.length); + assertTrue(lockFiles[0].endsWith(snapshotId2.getUUID() + ".v2_lock")); logger.info("--> make sure the node's repository can resolve the snapshots"); final List originalSnapshots = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); @@ -230,8 +230,8 @@ public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException { final SnapshotId snapshotId = snapshotInfo.snapshotId(); String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId.getUUID() + ".lock"); + assertEquals("there should be only one lock file, but found " + Arrays.toString(lockFiles), 1, lockFiles.length); + assertTrue(lockFiles[0].endsWith(snapshotId.getUUID() + ".v2_lock")); final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); @@ -305,8 +305,8 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { final SnapshotId snapshotId1 = snapshotInfo.snapshotId(); String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "lock files are " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId1.getUUID() + ".lock"); + assertEquals("lock files are " + Arrays.toString(lockFiles), 1, lockFiles.length); + assertTrue(lockFiles[0].endsWith(snapshotId1.getUUID() + ".v2_lock")); logger.info("--> create second remote index shallow snapshot"); snapshotInfo = createSnapshot( @@ -317,10 +317,10 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { final SnapshotId snapshotId2 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 2) : "lock files are " + Arrays.toString(lockFiles); + assertEquals("lock files are " + Arrays.toString(lockFiles), 2, lockFiles.length); List shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2); for (SnapshotId snapshotId : shallowCopySnapshotIDs) { - assert lockFiles[0].contains(snapshotId.getUUID()) || lockFiles[1].contains(snapshotId.getUUID()); + assertTrue(lockFiles[0].contains(snapshotId.getUUID()) || lockFiles[1].contains(snapshotId.getUUID())); } logger.info("--> create third remote index shallow snapshot"); snapshotInfo = createSnapshot( @@ -331,12 +331,14 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { final SnapshotId snapshotId3 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 3); + assertEquals(3, lockFiles.length); shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); for (SnapshotId snapshotId : shallowCopySnapshotIDs) { - assert lockFiles[0].contains(snapshotId.getUUID()) - || lockFiles[1].contains(snapshotId.getUUID()) - || lockFiles[2].contains(snapshotId.getUUID()); + assertTrue( + lockFiles[0].contains(snapshotId.getUUID()) + || lockFiles[1].contains(snapshotId.getUUID()) + || lockFiles[2].contains(snapshotId.getUUID()) + ); } logger.info("--> create normal snapshot"); createRepository(client, snapshotRepositoryName, snapshotRepoSettings); @@ -348,12 +350,14 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { final SnapshotId snapshotId4 = snapshotInfo.snapshotId(); lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 3) : "lock files are " + Arrays.toString(lockFiles); + assertEquals("lock files are " + Arrays.toString(lockFiles), 3, lockFiles.length); shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); for (SnapshotId snapshotId : shallowCopySnapshotIDs) { - assert lockFiles[0].contains(snapshotId.getUUID()) - || lockFiles[1].contains(snapshotId.getUUID()) - || lockFiles[2].contains(snapshotId.getUUID()); + assertTrue( + lockFiles[0].contains(snapshotId.getUUID()) + || lockFiles[1].contains(snapshotId.getUUID()) + || lockFiles[2].contains(snapshotId.getUUID()) + ); } logger.info("--> make sure the node's repository can resolve the snapshots"); From dad525aefaab01b8452f9db8d7fba70a4d3b5cc8 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 5 Oct 2023 19:58:58 -0400 Subject: [PATCH 122/388] Allow customization of netty channel handles before and during decompression (#10261) --- CHANGELOG.md | 1 + .../http/netty4/Netty4HeaderVerifierIT.java | 75 +++++++++++ .../transport/Netty4BlockingPlugin.java | 127 ++++++++++++++++++ .../netty4/Netty4HttpServerTransport.java | 30 ++++- .../transport/Netty4ModulePlugin.java | 2 +- .../java/org/opensearch/rest/RestHandler.java | 2 +- 6 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HeaderVerifierIT.java create mode 100644 modules/transport-netty4/src/internalClusterTest/java/org/opensearch/transport/Netty4BlockingPlugin.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ea8d5ae4cfe4..e408df2307587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562)) - Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204)) - [Remote Store] Add support to restrict creation & deletion if system repository and mutation of immutable settings of system repository ([#9839](https://github.com/opensearch-project/OpenSearch/pull/9839)) +- Improve compressed request handling ([#10261](https://github.com/opensearch-project/OpenSearch/pull/10261)) ### Dependencies - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HeaderVerifierIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HeaderVerifierIT.java new file mode 100644 index 0000000000000..c39567a005fd1 --- /dev/null +++ b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HeaderVerifierIT.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.http.netty4; + +import org.opensearch.OpenSearchNetty4IntegTestCase; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.http.HttpServerTransport; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.transport.Netty4BlockingPlugin; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import io.netty.buffer.ByteBufUtil; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http2.HttpConversionUtil; +import io.netty.util.ReferenceCounted; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static io.netty.handler.codec.http.HttpHeaderNames.HOST; + +@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) +public class Netty4HeaderVerifierIT extends OpenSearchNetty4IntegTestCase { + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(Netty4BlockingPlugin.class); + } + + public void testThatNettyHttpServerRequestBlockedWithHeaderVerifier() throws Exception { + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); + TransportAddress transportAddress = randomFrom(boundAddresses); + + final FullHttpRequest blockedRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"); + blockedRequest.headers().add("blockme", "Not Allowed"); + blockedRequest.headers().add(HOST, "localhost"); + blockedRequest.headers().add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), "http"); + + final List responses = new ArrayList<>(); + try (Netty4HttpClient nettyHttpClient = Netty4HttpClient.http2()) { + try { + FullHttpResponse blockedResponse = nettyHttpClient.send(transportAddress.address(), blockedRequest); + responses.add(blockedResponse); + String blockedResponseContent = new String(ByteBufUtil.getBytes(blockedResponse.content()), StandardCharsets.UTF_8); + assertThat(blockedResponseContent, containsString("Hit header_verifier")); + assertThat(blockedResponse.status().code(), equalTo(401)); + } finally { + responses.forEach(ReferenceCounted::release); + } + } + } + +} diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/transport/Netty4BlockingPlugin.java b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/transport/Netty4BlockingPlugin.java new file mode 100644 index 0000000000000..d5fe49952add3 --- /dev/null +++ b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/transport/Netty4BlockingPlugin.java @@ -0,0 +1,127 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.transport; + +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.http.HttpServerTransport; +import org.opensearch.http.netty4.Netty4HttpServerTransport; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.threadpool.ThreadPool; + +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.util.ReferenceCountUtil; + +public class Netty4BlockingPlugin extends Netty4ModulePlugin { + + public class Netty4BlockingHttpServerTransport extends Netty4HttpServerTransport { + + public Netty4BlockingHttpServerTransport( + Settings settings, + NetworkService networkService, + BigArrays bigArrays, + ThreadPool threadPool, + NamedXContentRegistry xContentRegistry, + Dispatcher dispatcher, + ClusterSettings clusterSettings, + SharedGroupFactory sharedGroupFactory, + Tracer tracer + ) { + super( + settings, + networkService, + bigArrays, + threadPool, + xContentRegistry, + dispatcher, + clusterSettings, + sharedGroupFactory, + tracer + ); + } + + @Override + protected ChannelInboundHandlerAdapter createHeaderVerifier() { + return new ExampleBlockingNetty4HeaderVerifier(); + } + } + + @Override + public Map> getHttpTransports( + Settings settings, + ThreadPool threadPool, + BigArrays bigArrays, + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedXContentRegistry xContentRegistry, + NetworkService networkService, + HttpServerTransport.Dispatcher dispatcher, + ClusterSettings clusterSettings, + Tracer tracer + ) { + return Collections.singletonMap( + NETTY_HTTP_TRANSPORT_NAME, + () -> new Netty4BlockingHttpServerTransport( + settings, + networkService, + bigArrays, + threadPool, + xContentRegistry, + dispatcher, + clusterSettings, + getSharedGroupFactory(settings), + tracer + ) + ); + } + + /** POC for how an external header verifier would be implemented */ + public class ExampleBlockingNetty4HeaderVerifier extends SimpleChannelInboundHandler { + + @Override + public void channelRead0(ChannelHandlerContext ctx, DefaultHttpRequest msg) throws Exception { + ReferenceCountUtil.retain(msg); + if (isBlocked(msg)) { + ByteBuf buf = Unpooled.copiedBuffer("Hit header_verifier".getBytes(StandardCharsets.UTF_8)); + final FullHttpResponse response = new DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.UNAUTHORIZED, buf); + ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE); + ReferenceCountUtil.release(msg); + } else { + // Lets the request pass to the next channel handler + ctx.fireChannelRead(msg); + } + } + + private boolean isBlocked(HttpRequest request) { + final boolean shouldBlock = request.headers().contains("blockme"); + + return shouldBlock; + } + } +} diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java index 0271472125814..1677f333a4b1c 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java @@ -334,7 +334,7 @@ public ChannelHandler configureServerChannelHandler() { return new HttpChannelHandler(this, handlingSettings); } - protected static final AttributeKey HTTP_CHANNEL_KEY = AttributeKey.newInstance("opensearch-http-channel"); + public static final AttributeKey HTTP_CHANNEL_KEY = AttributeKey.newInstance("opensearch-http-channel"); protected static final AttributeKey HTTP_SERVER_CHANNEL_KEY = AttributeKey.newInstance( "opensearch-http-server-channel" ); @@ -419,8 +419,8 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpMessage msg) throws E // If this handler is hit then no upgrade has been attempted and the client is just talking HTTP final ChannelPipeline pipeline = ctx.pipeline(); pipeline.addAfter(ctx.name(), "handler", getRequestHandler()); - pipeline.replace(this, "decoder_compress", new HttpContentDecompressor()); - + pipeline.replace(this, "header_verifier", transport.createHeaderVerifier()); + pipeline.addAfter("header_verifier", "decoder_compress", transport.createDecompressor()); pipeline.addAfter("decoder_compress", "aggregator", aggregator); if (handlingSettings.isCompression()) { pipeline.addAfter( @@ -446,7 +446,8 @@ protected void configureDefaultHttpPipeline(ChannelPipeline pipeline) { ); decoder.setCumulator(ByteToMessageDecoder.COMPOSITE_CUMULATOR); pipeline.addLast("decoder", decoder); - pipeline.addLast("decoder_compress", new HttpContentDecompressor()); + pipeline.addLast("header_verifier", transport.createHeaderVerifier()); + pipeline.addLast("decoder_compress", transport.createDecompressor()); pipeline.addLast("encoder", new HttpResponseEncoder()); final HttpObjectAggregator aggregator = new HttpObjectAggregator(handlingSettings.getMaxContentLength()); aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); @@ -487,13 +488,13 @@ protected void initChannel(Channel childChannel) throws Exception { final HttpObjectAggregator aggregator = new HttpObjectAggregator(handlingSettings.getMaxContentLength()); aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); - childChannel.pipeline() .addLast(new LoggingHandler(LogLevel.DEBUG)) .addLast(new Http2StreamFrameToHttpObjectCodec(true)) .addLast("byte_buf_sizer", byteBufSizer) .addLast("read_timeout", new ReadTimeoutHandler(transport.readTimeoutMillis, TimeUnit.MILLISECONDS)) - .addLast("decoder_decompress", new HttpContentDecompressor()); + .addLast("header_verifier", transport.createHeaderVerifier()) + .addLast("decoder_decompress", transport.createDecompressor()); if (handlingSettings.isCompression()) { childChannel.pipeline() @@ -531,4 +532,21 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { } } } + + /** + * Extension point that allows a NetworkPlugin to extend the netty pipeline and inspect headers after request decoding + */ + protected ChannelInboundHandlerAdapter createHeaderVerifier() { + // pass-through + return new ChannelInboundHandlerAdapter(); + } + + /** + * Extension point that allows a NetworkPlugin to override the default netty HttpContentDecompressor and supply a custom decompressor. + * + * Used in instances to conditionally decompress depending on the outcome from header verification + */ + protected ChannelInboundHandlerAdapter createDecompressor() { + return new HttpContentDecompressor(); + } } diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java index 1a34dfd2c9ee4..2bc795d11ed5d 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java @@ -144,7 +144,7 @@ public Map> getHttpTransports( ); } - private SharedGroupFactory getSharedGroupFactory(Settings settings) { + SharedGroupFactory getSharedGroupFactory(Settings settings) { SharedGroupFactory groupFactory = this.groupFactory.get(); if (groupFactory != null) { assert groupFactory.getSettings().equals(settings) : "Different settings than originally provided"; diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java index 7832649e8ad32..294dc3ffbe329 100644 --- a/server/src/main/java/org/opensearch/rest/RestHandler.java +++ b/server/src/main/java/org/opensearch/rest/RestHandler.java @@ -108,7 +108,7 @@ default List replacedRoutes() { } /** - * Controls whether requests handled by this class are allowed to to access system indices by default. + * Controls whether requests handled by this class are allowed to access system indices by default. * @return {@code true} if requests handled by this class should be allowed to access system indices. */ default boolean allowSystemIndexAccessByDefault() { From e4c477a68b0fa9272169325787088a8633d338fd Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 5 Oct 2023 21:52:24 -0400 Subject: [PATCH 123/388] Disable build cache since Github Action runners fail with 'no space left on device' (#10409) Signed-off-by: Andriy Redko --- .github/workflows/assemble.yml | 26 ++++++++++++++++++++++++++ .github/workflows/precommit.yml | 11 +---------- settings.gradle | 4 +++- 3 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/assemble.yml diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml new file mode 100644 index 0000000000000..6a66ac5fb5609 --- /dev/null +++ b/.github/workflows/assemble.yml @@ -0,0 +1,26 @@ +name: Gradle Assemble +on: [pull_request] + +jobs: + assemble: + if: github.repository == 'opensearch-project/OpenSearch' + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: 11 + distribution: temurin + - name: Setup docker (missing on MacOS) + if: runner.os == 'macos' + run: | + brew install docker + colima start + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + - name: Run Gradle (assemble) + run: | + ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index f4622859916c7..b04f404b11c55 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -1,4 +1,4 @@ -name: Gradle Precommit and Assemble +name: Gradle Precommit on: [pull_request] jobs: @@ -19,12 +19,3 @@ jobs: - name: Run Gradle (precommit) run: | ./gradlew javadoc precommit --parallel - - name: Setup docker (missing on MacOS) - if: runner.os == 'macos' - run: | - brew install docker - colima start - sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - - name: Run Gradle (assemble) - run: | - ./gradlew assemble --parallel diff --git a/settings.gradle b/settings.gradle index c04b5997d49b1..13cc6669e3d33 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,9 +13,11 @@ plugins { id "com.gradle.enterprise" version "3.14.1" } +ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE') + buildCache { local { - enabled = true + enabled = !disableBuildCache removeUnusedEntriesAfterDays = 14 } } From 4fee40d847199495f19bf1209b12f67c11e84f54 Mon Sep 17 00:00:00 2001 From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:14:05 +0000 Subject: [PATCH 124/388] Fix to include static settings in addition to dynamic repo metadata settings during s3 plugin reload (#10452) Signed-off-by: vikasvb90 --- .../java/org/opensearch/repositories/s3/S3Repository.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java index a69309d9e7a6f..bbc3451be1535 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java @@ -374,9 +374,8 @@ public void reload(RepositoryMetadata newRepositoryMetadata) { readRepositoryMetadata(); // Reload configs for S3RepositoryPlugin - final Map clientsSettings = S3ClientSettings.load(metadata.settings(), pluginConfigPath); - service.refreshAndClearCache(clientsSettings); - s3AsyncService.refreshAndClearCache(clientsSettings); + service.settings(metadata); + s3AsyncService.settings(metadata); // Reload configs for S3BlobStore BlobStore blobStore = getBlobStore(); From f4c7ac9fd87e1592a7cf4adf89594c176795409a Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 6 Oct 2023 10:52:36 -0400 Subject: [PATCH 125/388] Update Gradle to 8.4 (#10138) Signed-off-by: Andriy Redko --- .../gradle/internal/InternalBwcGitPlugin.java | 4 +--- gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew | 14 +++++++------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalBwcGitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalBwcGitPlugin.java index 159270d28e3d6..c6e49dc44d6bd 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalBwcGitPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalBwcGitPlugin.java @@ -76,7 +76,7 @@ public InternalBwcGitPlugin(ProviderFactory providerFactory, ExecOperations exec public void apply(Project project) { this.project = project; this.gitExtension = project.getExtensions().create("bwcGitConfig", BwcGitExtension.class); - Provider remote = providerFactory.systemProperty("bwc.remote").forUseAtConfigurationTime().orElse("opensearch-project"); + Provider remote = providerFactory.systemProperty("bwc.remote").orElse("opensearch-project"); TaskContainer tasks = project.getTasks(); TaskProvider createCloneTaskProvider = tasks.register("createClone", LoggedExec.class, createClone -> { @@ -105,7 +105,6 @@ public void apply(Project project) { String remoteRepo = remote.get(); // for testing only we can override the base remote url String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo") - .forUseAtConfigurationTime() .getOrElse("https://github.com/" + remoteRepo + "/OpenSearch.git"); addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl)); }); @@ -113,7 +112,6 @@ public void apply(Project project) { TaskProvider fetchLatestTaskProvider = tasks.register("fetchLatest", LoggedExec.class, fetchLatest -> { Provider gitFetchLatest = project.getProviders() .systemProperty("tests.bwc.git_fetch_latest") - .forUseAtConfigurationTime() .orElse("true") .map(fetchProp -> { if ("true".equals(fetchProp)) { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f01f0a84a786a..adfb521550eb9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e +distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419 diff --git a/gradlew b/gradlew index 0adc8e1a53214..1aa94a4269074 100755 --- a/gradlew +++ b/gradlew @@ -145,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -153,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -202,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ From cd34fc72d71cb0ff21475e1fe3ee36f66ed2e9a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:27:26 -0400 Subject: [PATCH 126/388] Bump com.google.api.grpc:proto-google-common-protos from 2.25.0 to 2.25.1 in /plugins/repository-gcs (#10298) * Bump com.google.api.grpc:proto-google-common-protos Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.25.0 to 2.25.1. - [Release notes](https://github.com/googleapis/sdk-platform-java/releases) - [Changelog](https://github.com/googleapis/sdk-platform-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/sdk-platform-java/commits) --- updated-dependencies: - dependency-name: com.google.api.grpc:proto-google-common-protos dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- plugins/repository-gcs/build.gradle | 2 +- .../licenses/proto-google-common-protos-2.25.0.jar.sha1 | 1 - .../licenses/proto-google-common-protos-2.25.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/proto-google-common-protos-2.25.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e408df2307587..4a039526d3664 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,7 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752)) - Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126)) - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.5 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206), [#10299](https://github.com/opensearch-project/OpenSearch/pull/10299)) -- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208)) +- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298)) - Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209)) - Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` - Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))` diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index ec040a180876e..52e5e71618d69 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -60,7 +60,7 @@ dependencies { api 'com.google.api-client:google-api-client:2.2.0' - api 'com.google.api.grpc:proto-google-common-protos:2.25.0' + api 'com.google.api.grpc:proto-google-common-protos:2.25.1' api 'com.google.api.grpc:proto-google-iam-v1:0.12.0' api "com.google.auth:google-auth-library-credentials:${versions.google_auth}" diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 deleted file mode 100644 index b5ef7ee78e794..0000000000000 --- a/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -adfa7c3d9b806969db75cf35fe4b286b3b8b1ce0 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.1.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.1.jar.sha1 new file mode 100644 index 0000000000000..cd065dabb8e8a --- /dev/null +++ b/plugins/repository-gcs/licenses/proto-google-common-protos-2.25.1.jar.sha1 @@ -0,0 +1 @@ +cb90049537b621e39610a110c58ce0b914ee3cc5 \ No newline at end of file From 9cb4f9dbd44e42269fe573f9592d58d4115cc522 Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Fri, 6 Oct 2023 14:31:14 -0700 Subject: [PATCH 127/388] Mute flaky concurrent segment search tests pending fixes (#10437) * DiversifiedSamplerIT.testNestedSamples * QueryProfilePhaseTests.testMaxScore * QueryProfilePhaseTests.testCollapseQuerySearchResults * HighlighterSearchIT.testHighlightQueryRewriteDatesWithNow * FieldCapabilitiesIT.testWithIndexFilter * QueryProfilePhaseTests.testDisableTopScoreCollection Signed-off-by: Jay Deng Signed-off-by: Jay Deng --- .../aggregations/bucket/DiversifiedSamplerIT.java | 4 ++++ .../subphase/highlight/HighlighterSearchIT.java | 4 ++++ .../search/fieldcaps/FieldCapabilitiesIT.java | 4 ++++ .../search/query/QueryProfilePhaseTests.java | 12 ++++++++++++ 4 files changed, 24 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java index 5e95073209c71..865dd670fbf68 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java @@ -221,6 +221,10 @@ public void testNestedDiversity() throws Exception { } public void testNestedSamples() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10046", + internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + ); // Test samples nested under samples int MAX_DOCS_PER_AUTHOR = 1; int MAX_DOCS_PER_GENRE = 2; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 4cdf5ae8e674f..42d91ac945662 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -3343,6 +3343,10 @@ public void testFiltersFunctionScoreQueryHighlight() throws Exception { } public void testHighlightQueryRewriteDatesWithNow() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10434", + internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + ); assertAcked( client().admin() .indices() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java index f5d1b8234558e..6b95405b3ebd4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -244,6 +244,10 @@ public void testWithIndexAlias() { } public void testWithIndexFilter() throws InterruptedException { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10433", + internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + ); assertAcked(prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword")); assertAcked(prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long")); diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index fc77c1b356124..92d27032b62e3 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -1066,6 +1066,10 @@ public void testIndexSortScrollOptimization() throws Exception { } public void testDisableTopScoreCollection() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10469", + executor != null + ); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); @@ -1237,6 +1241,10 @@ public void testMinScore() throws Exception { } public void testMaxScore() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/9932", + executor != null + ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("filter", SortField.Type.STRING)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); @@ -1356,6 +1364,10 @@ public void testMaxScore() throws Exception { } public void testCollapseQuerySearchResults() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10139", + executor != null + ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("user", SortField.Type.INT)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); From 2bf63c931356d6a2e31ea00507032c2cd115af45 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 6 Oct 2023 23:14:25 -0400 Subject: [PATCH 128/388] Bump com.google.api.grpc:proto-google-common-protos from 2.25.0 to 2.25.1 in /plugins/repository-gcs (#10298) (#10461) * Bump com.google.api.grpc:proto-google-common-protos Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.25.0 to 2.25.1. - [Release notes](https://github.com/googleapis/sdk-platform-java/releases) - [Changelog](https://github.com/googleapis/sdk-platform-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/sdk-platform-java/commits) --- updated-dependencies: - dependency-name: com.google.api.grpc:proto-google-common-protos dependency-type: direct:production update-type: version-update:semver-patch ... * Updating SHAs * Update changelog --------- Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .ci/bwcVersions | 2 ++ libs/core/src/main/java/org/opensearch/Version.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 6b86da2c91261..cfaadc5ed1e5e 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -23,4 +23,6 @@ BWC_VERSION: - "2.9.0" - "2.9.1" - "2.10.0" + - "2.10.1" - "2.11.0" + - "2.12.0" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 32f4ca0317907..3c210e5c7cf96 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -94,7 +94,9 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_9_1 = new Version(2090199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_2_10_1 = new Version(2100199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0); public static final Version CURRENT = V_3_0_0; From 98defa595cc83369fa863d50057d518133b81540 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Sat, 7 Oct 2023 13:40:38 +0530 Subject: [PATCH 129/388] [Remote Store] Using hash of node id in metadata file names (#10480) Signed-off-by: Gaurav Bafna --- .../org/opensearch/index/remote/RemoteStoreUtils.java | 7 +++---- .../index/store/RemoteSegmentStoreDirectory.java | 3 ++- .../translog/transfer/TranslogTransferMetadata.java | 2 +- .../opensearch/index/remote/RemoteStoreUtilsTests.java | 6 +++--- .../index/store/RemoteSegmentStoreDirectoryTests.java | 9 ++++++--- .../translog/transfer/TranslogTransferManagerTests.java | 8 ++++++-- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index 0ca9e0209c5ec..b4c33d781af86 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -82,7 +82,7 @@ public static String getSegmentName(String filename) { * @param fn Function to extract PrimaryTerm_Generation and Node Id from metadata file name . * fn returns null if node id is not part of the file name */ - static public void verifyNoMultipleWriters(List mdFiles, Function> fn) { + public static void verifyNoMultipleWriters(List mdFiles, Function> fn) { Map nodesByPrimaryTermAndGen = new HashMap<>(); mdFiles.forEach(mdFile -> { Tuple nodeIdByPrimaryTermAndGen = fn.apply(mdFile); @@ -91,10 +91,9 @@ static public void verifyNoMultipleWriters(List mdFiles, Function bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(mdFilename, 1)); @@ -167,7 +167,7 @@ public void testVerifyMultipleWriters_Translog() throws InterruptedException { bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(mdFilename, 1)); - TranslogTransferMetadata tm3 = new TranslogTransferMetadata(1, 1, 1, 2, "node-2"); + TranslogTransferMetadata tm3 = new TranslogTransferMetadata(1, 1, 1, 2, "node--2"); bmList.add(new PlainBlobMetadata(tm3.getFileName(), 1)); List finalBmList = bmList; assertThrows( diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 4a89b3c718f0b..0f44d5c3b2f53 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -67,6 +67,7 @@ import org.mockito.Mockito; import static org.opensearch.index.store.RemoteSegmentStoreDirectory.METADATA_FILES_TO_FETCH; +import static org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR; import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes; import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata; import static org.hamcrest.CoreMatchers.is; @@ -213,9 +214,7 @@ public void testUploadedSegmentMetadataFromStringException() { } public void testGetPrimaryTermGenerationUuid() { - String[] filenameTokens = "abc__9223372036854775795__9223372036854775784__uuid_xyz".split( - RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR - ); + String[] filenameTokens = "abc__9223372036854775795__9223372036854775784__uuid_xyz".split(SEPARATOR); assertEquals(12, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getPrimaryTerm(filenameTokens)); assertEquals(23, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getGeneration(filenameTokens)); } @@ -1178,6 +1177,10 @@ public void testMetadataFileNameOrder() { actualList.sort(String::compareTo); assertEquals(List.of(file3, file2, file4, file6, file5, file1), actualList); + + long count = file1.chars().filter(ch -> ch == SEPARATOR.charAt(0)).count(); + // There should not be any `_` in mdFile name as it is used a separator . + assertEquals(14, count); } private static class WrapperIndexOutput extends IndexOutput { diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index a48dbdcdacb71..5deb4c83e52fc 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -44,6 +44,7 @@ import org.mockito.Mockito; +import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anySet; @@ -515,10 +516,13 @@ public void testMetadataConflict() throws InterruptedException { null, remoteTranslogTransferTracker ); - TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2, "node-1"); + TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2, "node--1"); String mdFilename = tm.getFileName(); + long count = mdFilename.chars().filter(ch -> ch == METADATA_SEPARATOR.charAt(0)).count(); + // There should not be any `_` in mdFile name as it is used a separator . + assertEquals(10, count); Thread.sleep(1); - TranslogTransferMetadata tm2 = new TranslogTransferMetadata(1, 1, 1, 2, "node-2"); + TranslogTransferMetadata tm2 = new TranslogTransferMetadata(1, 1, 1, 2, "node--2"); String mdFilename2 = tm2.getFileName(); doAnswer(invocation -> { From 732ce21a3e0ab5492d5cc327d287525b0170234e Mon Sep 17 00:00:00 2001 From: Ashish Date: Sat, 7 Oct 2023 14:12:36 +0530 Subject: [PATCH 130/388] Fix flaky testRemoteRefreshRetryOnFailure in RemoteStoreRefreshListenerIT (#10464) Signed-off-by: Ashish Singh --- .../remotestore/RemoteStoreRefreshListenerIT.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java index 88760b7bbfad2..acdb21d072320 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java @@ -28,9 +28,13 @@ public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockRepositoryIntegTestCase { public void testRemoteRefreshRetryOnFailure() throws Exception { - Path location = randomRepoPath().toAbsolutePath(); setup(location, randomDoubleBetween(0.1, 0.15, true), "metadata", 10L); + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey(), false)) + .get(); // Here we are having flush/refresh after each iteration of indexing. However, the refresh will not always succeed // due to IOExceptions that are thrown while doing uploadBlobs. From 22ddcf19c7a0ceb8998cdc6344ba132b547cef1c Mon Sep 17 00:00:00 2001 From: Ashish Date: Sat, 7 Oct 2023 15:48:09 +0530 Subject: [PATCH 131/388] Fix failing testGetPrimaryTermAndGeneration in TranslogTransferManagerTests (#10490) Signed-off-by: Ashish Singh --- .../transfer/TranslogTransferManagerTests.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 5deb4c83e52fc..af596e7df02c2 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -39,7 +39,9 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Objects; import java.util.Set; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import org.mockito.Mockito; @@ -504,8 +506,12 @@ private void assertTlogCkpDownloadStats() { } public void testGetPrimaryTermAndGeneration() { - String tm = new TranslogTransferMetadata(1, 2, 1, 2, "node-1").getFileName(); - assertEquals(new Tuple<>(new Tuple<>(1L, 2L), "node-1"), TranslogTransferMetadata.getNodeIdByPrimaryTermAndGeneration(tm)); + String nodeId = UUID.randomUUID().toString(); + String tm = new TranslogTransferMetadata(1, 2, 1, 2, nodeId).getFileName(); + Tuple, String> actualOutput = TranslogTransferMetadata.getNodeIdByPrimaryTermAndGeneration(tm); + assertEquals(1L, (long) (actualOutput.v1().v1())); + assertEquals(2L, (long) (actualOutput.v1().v2())); + assertEquals(String.valueOf(Objects.hash(nodeId)), actualOutput.v2()); } public void testMetadataConflict() throws InterruptedException { From 8bb11a68ec5c4fe784b2a1efdd459d2f6476cecc Mon Sep 17 00:00:00 2001 From: Ashish Date: Sat, 7 Oct 2023 22:58:13 +0530 Subject: [PATCH 132/388] Fix bugs causing red indexes with remote indexes during translog upload & store recovery (#10449) --------- Signed-off-by: Ashish Singh --- ...emoteStoreMockRepositoryIntegTestCase.java | 2 +- ...moteStoreBackpressureAndResiliencyIT.java} | 74 ++++++++++++++++++- .../org/opensearch/index/IndexService.java | 2 +- .../opensearch/index/shard/IndexShard.java | 5 +- 4 files changed, 79 insertions(+), 4 deletions(-) rename server/src/internalClusterTest/java/org/opensearch/remotestore/{RemoteStoreBackpressureIT.java => RemoteStoreBackpressureAndResiliencyIT.java} (67%) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 9d4d8aa24bd51..2053800504c89 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -160,7 +160,7 @@ private String getLocalSegmentFilename(String remoteFilename) { return remoteFilename.split(RemoteSegmentStoreDirectory.SEGMENT_NAME_UUID_SEPARATOR)[0]; } - private IndexResponse indexSingleDoc() { + protected IndexResponse indexSingleDoc() { return client().prepareIndex(INDEX_NAME) .setId(UUIDs.randomBase64UUID()) .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java similarity index 67% rename from server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java rename to server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java index 3462054c23630..2c6db6ae19a9a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java @@ -12,12 +12,18 @@ import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.AbstractAsyncTask; +import org.opensearch.common.util.concurrent.UncategorizedExecutionException; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.index.IndexService; import org.opensearch.index.remote.RemoteSegmentTransferTracker; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.snapshots.mockstore.MockRepository; import org.opensearch.test.OpenSearchIntegTestCase; @@ -33,7 +39,7 @@ import static org.opensearch.index.remote.RemoteStorePressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class RemoteStoreBackpressureIT extends AbstractRemoteStoreMockRepositoryIntegTestCase { +public class RemoteStoreBackpressureAndResiliencyIT extends AbstractRemoteStoreMockRepositoryIntegTestCase { public void testWritesRejectedDueToConsecutiveFailureBreach() throws Exception { // Here the doc size of the request remains same throughout the test. After initial indexing, all remote store interactions // fail leading to consecutive failure limit getting exceeded and leading to rejections. @@ -156,4 +162,70 @@ private String generateString(int sizeInBytes) { sb.append("}"); return sb.toString(); } + + /** + * Fixes Github#10398 + */ + public void testAsyncTrimTaskSucceeds() { + Path location = randomRepoPath().toAbsolutePath(); + String dataNodeName = setup(location, 0d, "metadata", Long.MAX_VALUE); + + logger.info("Increasing the frequency of async trim task to ensure it runs in background while indexing"); + IndexService indexService = internalCluster().getInstance(IndicesService.class, dataNodeName).iterator().next(); + ((AbstractAsyncTask) indexService.getTrimTranslogTask()).setInterval(TimeValue.timeValueMillis(100)); + + logger.info("--> Indexing data"); + indexData(randomIntBetween(2, 5), true); + logger.info("--> Indexing succeeded"); + + MockRepository translogRepo = (MockRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) + .repository(TRANSLOG_REPOSITORY_NAME); + logger.info("--> Failing all remote store interaction"); + translogRepo.setRandomControlIOExceptionRate(1d); + + for (int i = 0; i < randomIntBetween(5, 10); i++) { + UncategorizedExecutionException exception = assertThrows(UncategorizedExecutionException.class, this::indexSingleDoc); + assertEquals("Failed execution", exception.getMessage()); + } + + translogRepo.setRandomControlIOExceptionRate(0d); + indexSingleDoc(); + logger.info("Indexed single doc successfully"); + } + + /** + * Fixes Github#10400 + */ + public void testSkipLoadGlobalCheckpointToReplicationTracker() { + Path location = randomRepoPath().toAbsolutePath(); + String dataNodeName = setup(location, 0d, "metadata", Long.MAX_VALUE); + + logger.info("--> Indexing data"); + indexData(randomIntBetween(1, 2), true); + logger.info("--> Indexing succeeded"); + + IndexService indexService = internalCluster().getInstance(IndicesService.class, dataNodeName).iterator().next(); + IndexShard indexShard = indexService.getShard(0); + indexShard.failShard("failing shard", null); + + ensureRed(INDEX_NAME); + + MockRepository translogRepo = (MockRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) + .repository(TRANSLOG_REPOSITORY_NAME); + logger.info("--> Failing all remote store interaction"); + translogRepo.setRandomControlIOExceptionRate(1d); + client().admin().cluster().prepareReroute().setRetryFailed(true).get(); + // CLuster stays red still as the remote interactions are still failing + ensureRed(INDEX_NAME); + + logger.info("Retrying to allocate failed shards"); + client().admin().cluster().prepareReroute().setRetryFailed(true).get(); + // CLuster stays red still as the remote interactions are still failing + ensureRed(INDEX_NAME); + + logger.info("Stop failing all remote store interactions"); + translogRepo.setRandomControlIOExceptionRate(0d); + client().admin().cluster().prepareReroute().setRetryFailed(true).get(); + ensureGreen(INDEX_NAME); + } } diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index df8e8070b8e03..af23145be9f89 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -1286,7 +1286,7 @@ AsyncTranslogFSync getFsyncTask() { // for tests return fsyncTask; } - AsyncTrimTranslogTask getTrimTranslogTask() { // for tests + public AsyncTrimTranslogTask getTrimTranslogTask() { // for tests return trimTranslogTask; } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 833c91c1766c8..251f9a5ae01c0 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1465,6 +1465,9 @@ public void flush(FlushRequest request) { * {@link org.opensearch.index.translog.TranslogDeletionPolicy} for details */ public void trimTranslog() { + if (isRemoteTranslogEnabled()) { + return; + } verifyNotClosed(); final Engine engine = getEngine(); engine.translogManager().trimUnreferencedTranslogFiles(); @@ -2314,7 +2317,7 @@ public void openEngineAndRecoverFromTranslog() throws IOException { }; // Do not load the global checkpoint if this is a remote snapshot index - if (indexSettings.isRemoteSnapshot() == false) { + if (indexSettings.isRemoteSnapshot() == false && indexSettings.isRemoteTranslogStoreEnabled() == false) { loadGlobalCheckpointToReplicationTracker(); } From 74ffdb6dba6b8e307a2c491cc2498750e5548b8e Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Mon, 9 Oct 2023 21:03:14 -0700 Subject: [PATCH 133/388] Avoid size=0 for terminate_after tests (#10436) Signed-off-by: Jay Deng --- .../search/simple/SimpleSearchIT.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index 95b36311f6b8b..7aae41d939cac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -283,10 +283,9 @@ public void testSimpleDateRange() throws Exception { assertHitCount(searchResponse, 2L); } - public void testSimpleTerminateAfterCount() throws Exception { + public void dotestSimpleTerminateAfterCountWithSize(int size, int max) throws Exception { prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); ensureGreen(); - int max = randomIntBetween(3, 29); List docbuilders = new ArrayList<>(max); for (int i = 1; i <= max; i++) { @@ -299,9 +298,7 @@ public void testSimpleTerminateAfterCount() throws Exception { refresh(); SearchResponse searchResponse; - int size; for (int i = 1; i < max; i++) { - size = randomIntBetween(0, max); searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max)) .setTerminateAfter(i) @@ -322,7 +319,18 @@ public void testSimpleTerminateAfterCount() throws Exception { assertFalse(searchResponse.isTerminatedEarly()); } - public void testSimpleTerminateAfterTrackTotalHitsUpTo() throws Exception { + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10435") + public void testSimpleTerminateAfterCountSize0() throws Exception { + int max = randomIntBetween(3, 29); + dotestSimpleTerminateAfterCountWithSize(0, max); + } + + public void testSimpleTerminateAfterCountRandomSize() throws Exception { + int max = randomIntBetween(3, 29); + dotestSimpleTerminateAfterCountWithSize(randomIntBetween(1, max), max); + } + + public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Exception { prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); ensureGreen(); int numDocs = 29; @@ -337,8 +345,6 @@ public void testSimpleTerminateAfterTrackTotalHitsUpTo() throws Exception { ensureGreen(); refresh(); - // size=0 is a special case where topDocsCollector is not added - int size = randomIntBetween(0, 1); SearchResponse searchResponse; searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) @@ -399,6 +405,15 @@ public void testSimpleTerminateAfterTrackTotalHitsUpTo() throws Exception { assertEquals(GREATER_THAN_OR_EQUAL_TO, searchResponse.getHits().getTotalHits().relation); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10435") + public void testSimpleTerminateAfterTrackTotalHitsUpToRandomSize() throws Exception { + doTestSimpleTerminateAfterTrackTotalHitsUpTo(0); + } + + public void testSimpleTerminateAfterTrackTotalHitsUpToSize0() throws Exception { + doTestSimpleTerminateAfterTrackTotalHitsUpTo(randomIntBetween(1, 29)); + } + public void testSimpleIndexSortEarlyTerminate() throws Exception { prepareCreate("test").setSettings( Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).put("index.sort.field", "rank") From 8e5e54e87f6f86a2425a5c767e208767ba3df14c Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 10 Oct 2023 16:16:36 +0530 Subject: [PATCH 134/388] Use file length from metadata while downloading segments from remote store (#10399) Signed-off-by: Sachin Kale --- .../index/store/RemoteDirectory.java | 13 +++-- .../store/RemoteSegmentStoreDirectory.java | 3 +- .../index/store/RemoteDirectoryTests.java | 54 +++++++++---------- .../RemoteSegmentStoreDirectoryTests.java | 5 +- 4 files changed, 39 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index eb75c39532d71..70a88f6159764 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -39,7 +39,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -193,10 +192,14 @@ public IndexOutput createOutput(String name, IOContext context) { */ @Override public IndexInput openInput(String name, IOContext context) throws IOException { + return openInput(name, fileLength(name), context); + } + + public IndexInput openInput(String name, long fileLength, IOContext context) throws IOException { InputStream inputStream = null; try { inputStream = blobContainer.readBlob(name); - return new RemoteIndexInput(name, downloadRateLimiter.apply(inputStream), fileLength(name)); + return new RemoteIndexInput(name, downloadRateLimiter.apply(inputStream), fileLength); } catch (Exception e) { // Incase the RemoteIndexInput creation fails, close the input stream to avoid file handler leak. if (inputStream != null) { @@ -230,9 +233,9 @@ public void close() throws IOException { @Override public long fileLength(String name) throws IOException { // ToDo: Instead of calling remote store each time, keep a cache with segment metadata - Map metadata = blobContainer.listBlobsByPrefix(name); - if (metadata.containsKey(name)) { - return metadata.get(name).length(); + List metadata = blobContainer.listBlobsByPrefixInSortedOrder(name, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC); + if (metadata.size() == 1 && metadata.get(0).name().equals(name)) { + return metadata.get(0).length(); } throw new NoSuchFileException(name); } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index dc9706306b408..c0b302c68c0ea 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -434,8 +434,9 @@ public IndexOutput createOutput(String name, IOContext context) throws IOExcepti @Override public IndexInput openInput(String name, IOContext context) throws IOException { String remoteFilename = getExistingRemoteFilename(name); + long fileLength = fileLength(name); if (remoteFilename != null) { - return remoteDataDirectory.openInput(remoteFilename, context); + return remoteDataDirectory.openInput(remoteFilename, fileLength, context); } else { throw new NoSuchFileException(name); } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java index 3740abc57b02d..9e38e1749d434 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java @@ -29,7 +29,6 @@ import java.nio.file.NoSuchFileException; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -41,11 +40,13 @@ import org.mockito.Mockito; +import static org.opensearch.common.blobstore.BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -204,13 +205,29 @@ public void testCreateOutput() { public void testOpenInput() throws IOException { InputStream mockInputStream = mock(InputStream.class); when(blobContainer.readBlob("segment_1")).thenReturn(mockInputStream); - Map fileInfo = new HashMap<>(); - fileInfo.put("segment_1", new PlainBlobMetadata("segment_1", 100)); - when(blobContainer.listBlobsByPrefix("segment_1")).thenReturn(fileInfo); + + BlobMetadata blobMetadata = new PlainBlobMetadata("segment_1", 100); + + when(blobContainer.listBlobsByPrefixInSortedOrder("segment_1", 1, LEXICOGRAPHIC)).thenReturn(List.of(blobMetadata)); IndexInput indexInput = remoteDirectory.openInput("segment_1", IOContext.DEFAULT); assertTrue(indexInput instanceof RemoteIndexInput); assertEquals(100, indexInput.length()); + verify(blobContainer).listBlobsByPrefixInSortedOrder("segment_1", 1, LEXICOGRAPHIC); + } + + public void testOpenInputWithLength() throws IOException { + InputStream mockInputStream = mock(InputStream.class); + when(blobContainer.readBlob("segment_1")).thenReturn(mockInputStream); + + BlobMetadata blobMetadata = new PlainBlobMetadata("segment_1", 100); + + when(blobContainer.listBlobsByPrefixInSortedOrder("segment_1", 1, LEXICOGRAPHIC)).thenReturn(List.of(blobMetadata)); + + IndexInput indexInput = remoteDirectory.openInput("segment_1", 100, IOContext.DEFAULT); + assertTrue(indexInput instanceof RemoteIndexInput); + assertEquals(100, indexInput.length()); + verify(blobContainer, times(0)).listBlobsByPrefixInSortedOrder("segment_1", 1, LEXICOGRAPHIC); } public void testOpenInputIOException() throws IOException { @@ -228,9 +245,8 @@ public void testOpenInputNoSuchFileException() throws IOException { } public void testFileLength() throws IOException { - Map fileInfo = new HashMap<>(); - fileInfo.put("segment_1", new PlainBlobMetadata("segment_1", 100)); - when(blobContainer.listBlobsByPrefix("segment_1")).thenReturn(fileInfo); + BlobMetadata blobMetadata = new PlainBlobMetadata("segment_1", 100); + when(blobContainer.listBlobsByPrefixInSortedOrder("segment_1", 1, LEXICOGRAPHIC)).thenReturn(List.of(blobMetadata)); assertEquals(100, remoteDirectory.fileLength("segment_1")); } @@ -246,13 +262,7 @@ public void testListFilesByPrefixInLexicographicOrder() throws IOException { LatchedActionListener> latchedActionListener = invocation.getArgument(3); latchedActionListener.onResponse(List.of(new PlainBlobMetadata("metadata_1", 1))); return null; - }).when(blobContainer) - .listBlobsByPrefixInSortedOrder( - eq("metadata"), - eq(1), - eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), - any(ActionListener.class) - ); + }).when(blobContainer).listBlobsByPrefixInSortedOrder(eq("metadata"), eq(1), eq(LEXICOGRAPHIC), any(ActionListener.class)); assertEquals(List.of("metadata_1"), remoteDirectory.listFilesByPrefixInLexicographicOrder("metadata", 1)); } @@ -262,13 +272,7 @@ public void testListFilesByPrefixInLexicographicOrderEmpty() throws IOException LatchedActionListener> latchedActionListener = invocation.getArgument(3); latchedActionListener.onResponse(List.of()); return null; - }).when(blobContainer) - .listBlobsByPrefixInSortedOrder( - eq("metadata"), - eq(1), - eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), - any(ActionListener.class) - ); + }).when(blobContainer).listBlobsByPrefixInSortedOrder(eq("metadata"), eq(1), eq(LEXICOGRAPHIC), any(ActionListener.class)); assertEquals(List.of(), remoteDirectory.listFilesByPrefixInLexicographicOrder("metadata", 1)); } @@ -278,13 +282,7 @@ public void testListFilesByPrefixInLexicographicOrderException() { LatchedActionListener> latchedActionListener = invocation.getArgument(3); latchedActionListener.onFailure(new IOException("Error")); return null; - }).when(blobContainer) - .listBlobsByPrefixInSortedOrder( - eq("metadata"), - eq(1), - eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), - any(ActionListener.class) - ); + }).when(blobContainer).listBlobsByPrefixInSortedOrder(eq("metadata"), eq(1), eq(LEXICOGRAPHIC), any(ActionListener.class)); assertThrows(IOException.class, () -> remoteDirectory.listFilesByPrefixInLexicographicOrder("metadata", 1)); } diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 0f44d5c3b2f53..3e45699ef36eb 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -71,6 +71,7 @@ import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes; import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata; import static org.hamcrest.CoreMatchers.is; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.contains; import static org.mockito.Mockito.any; @@ -391,7 +392,7 @@ public void testOpenInput() throws IOException { remoteSegmentStoreDirectory.init(); IndexInput indexInput = mock(IndexInput.class); - when(remoteDataDirectory.openInput(startsWith("_0.si"), eq(IOContext.DEFAULT))).thenReturn(indexInput); + when(remoteDataDirectory.openInput(startsWith("_0.si"), anyLong(), eq(IOContext.DEFAULT))).thenReturn(indexInput); assertEquals(indexInput, remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); } @@ -404,7 +405,7 @@ public void testOpenInputException() throws IOException { populateMetadata(); remoteSegmentStoreDirectory.init(); - when(remoteDataDirectory.openInput(startsWith("_0.si"), eq(IOContext.DEFAULT))).thenThrow(new IOException("Error")); + when(remoteDataDirectory.openInput(startsWith("_0.si"), anyLong(), eq(IOContext.DEFAULT))).thenThrow(new IOException("Error")); assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); } From 506065c91f055cc1f925180afc1005c6989b7418 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 10 Oct 2023 08:54:50 -0500 Subject: [PATCH 135/388] Fix ScalingThreadPoolTest to handle new pool (#10427) This was a bad use of randomization. The test is super fast so it can be run against every scaling thread pool every time. Signed-off-by: Andrew Ross --- .../threadpool/ScalingThreadPoolTests.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java index 92bd15d818bca..ba2d4b8c247bb 100644 --- a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java @@ -32,10 +32,13 @@ package org.opensearch.threadpool; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -43,14 +46,29 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.equalTo; public class ScalingThreadPoolTests extends OpenSearchThreadPoolTestCase { + @ParametersFactory + public static Collection scalingThreadPools() { + return ThreadPool.THREAD_POOL_TYPES.entrySet() + .stream() + .filter(t -> t.getValue().equals(ThreadPool.ThreadPoolType.SCALING)) + .map(e -> new String[] { e.getKey() }) + .collect(Collectors.toList()); + } + + private final String threadPoolName; + + public ScalingThreadPoolTests(String threadPoolName) { + this.threadPoolName = threadPoolName; + } + public void testScalingThreadPoolConfiguration() throws InterruptedException { - final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING); final Settings.Builder builder = Settings.builder(); final int core; @@ -136,11 +154,11 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso sizes.put(ThreadPool.Names.TRANSLOG_SYNC, n -> 4 * n); sizes.put(ThreadPool.Names.REMOTE_PURGE, ThreadPool::halfAllocatedProcessorsMaxFive); sizes.put(ThreadPool.Names.REMOTE_REFRESH_RETRY, ThreadPool::halfAllocatedProcessorsMaxTen); + sizes.put(ThreadPool.Names.REMOTE_RECOVERY, ThreadPool::halfAllocatedProcessorsMaxTen); return sizes.get(threadPoolName).apply(numberOfProcessors); } public void testScalingThreadPoolIsBounded() throws InterruptedException { - final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING); final int size = randomIntBetween(32, 512); final Settings settings = Settings.builder().put("thread_pool." + threadPoolName + ".max", size).build(); runScalingThreadPoolTest(settings, (clusterSettings, threadPool) -> { @@ -170,7 +188,6 @@ public void testScalingThreadPoolIsBounded() throws InterruptedException { } public void testScalingThreadPoolThreadsAreTerminatedAfterKeepAlive() throws InterruptedException { - final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING); final int min = "generic".equals(threadPoolName) ? 4 : 1; final Settings settings = Settings.builder() .put("thread_pool." + threadPoolName + ".max", 128) From 562e3b2503a3892df2fa849792bf8ab89ce094ca Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 10 Oct 2023 10:22:18 -0500 Subject: [PATCH 136/388] Fix flaky SearchScrollWithFailingNodesIT (#10374) The test intended to stop a data node and called a method named `stopRandomNonClusterManagerNode()` in order to do that. However, that method would stop a random node that was not the currently elected cluster manager, regardless of node role. I have also renamed that method hoping to be more clear. Resolves #10137 Signed-off-by: Andrew Ross --- .../cluster/MinimumClusterManagerNodesIT.java | 4 ++-- .../opensearch/discovery/DiscoveryDisruptionIT.java | 2 +- .../search/scroll/SearchScrollWithFailingNodesIT.java | 2 +- .../java/org/opensearch/test/InternalTestCluster.java | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java index 4c8bf24b1655a..84648eda3d38c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java @@ -317,8 +317,8 @@ public void testThreeNodesNoClusterManagerBlock() throws Exception { ); Settings nonClusterManagerDataPathSettings1 = internalCluster().dataPathSettings(nonClusterManagerNodes.get(0)); Settings nonClusterManagerDataPathSettings2 = internalCluster().dataPathSettings(nonClusterManagerNodes.get(1)); - internalCluster().stopRandomNonClusterManagerNode(); - internalCluster().stopRandomNonClusterManagerNode(); + internalCluster().stopRandomNodeNotCurrentClusterManager(); + internalCluster().stopRandomNodeNotCurrentClusterManager(); logger.info("--> verify that there is no cluster-manager anymore on remaining node"); // spin here to wait till the state is set diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/DiscoveryDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/DiscoveryDisruptionIT.java index a2864b6dfd1da..70124c8c46700 100644 --- a/server/src/internalClusterTest/java/org/opensearch/discovery/DiscoveryDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/discovery/DiscoveryDisruptionIT.java @@ -136,7 +136,7 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception { // shutting down the nodes, to avoid the leakage check tripping // on the states associated with the commit requests we may have dropped - internalCluster().stopRandomNonClusterManagerNode(); + internalCluster().stopRandomNodeNotCurrentClusterManager(); } public void testClusterFormingWithASlowNode() { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java index f16b9a4d67b49..27002b844da1d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java @@ -119,7 +119,7 @@ public void testScanScrollWithShardExceptions() throws Exception { assertThat(numHits, equalTo(100L)); clearScroll("_all"); - internalCluster().stopRandomNonClusterManagerNode(); + internalCluster().stopRandomDataNode(); searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(10).setScroll(TimeValue.timeValueMinutes(1)).get(); assertThat(searchResponse.getSuccessfulShards(), lessThan(searchResponse.getTotalShards())); diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index d3e24ccd90500..acdc5b94804c6 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -1816,7 +1816,7 @@ public synchronized void stopCurrentClusterManagerNode() throws IOException { /** * Stops any of the current nodes but not the cluster-manager node. */ - public synchronized void stopRandomNonClusterManagerNode() throws IOException { + public synchronized void stopRandomNodeNotCurrentClusterManager() throws IOException { NodeAndClient nodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getClusterManagerName()).negate()); if (nodeAndClient != null) { logger.info( @@ -1841,11 +1841,11 @@ public synchronized void stopCurrentMasterNode() throws IOException { /** * Stops any of the current nodes but not the cluster-manager node. * - * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #stopRandomNonClusterManagerNode()} + * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #stopRandomNodeNotCurrentClusterManager()} */ @Deprecated - public synchronized void stopRandomNonMasterNode() throws IOException { - stopRandomNonClusterManagerNode(); + public synchronized void stopRandomNodeNotCurrentMaster() throws IOException { + stopRandomNodeNotCurrentClusterManager(); } /** @@ -1860,7 +1860,7 @@ public void stopAllNodes() { } int totalClusterManagerNodes = numClusterManagerNodes(); while (totalClusterManagerNodes > 1) { - stopRandomNonClusterManagerNode(); + stopRandomNodeNotCurrentClusterManager(); totalClusterManagerNodes -= 1; } stopCurrentClusterManagerNode(); From a43a16639530c50dc37fd486e67e2af999e82f65 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 11 Oct 2023 05:05:16 +0800 Subject: [PATCH 137/388] Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure (#10496) * Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure Signed-off-by: Gao Binlong * Modify changelog Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../ingest/SimulatePipelineRequestParsingTests.java | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a039526d3664..ba71604d1a94e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,6 +149,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194)) - Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256)) - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370)) +- Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496)) ### Security diff --git a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java index 04a9d08bb22bc..41f782e308785 100644 --- a/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -85,7 +85,7 @@ public void init() throws IOException { when(ingestService.getProcessorFactories()).thenReturn(registry); } - public void testParseUsingPipelineStore(boolean useExplicitType) throws Exception { + public void testParseUsingPipelineStore() throws Exception { int numDocs = randomIntBetween(1, 10); Map requestContent = new HashMap<>(); @@ -131,7 +131,7 @@ public void testParseUsingPipelineStore(boolean useExplicitType) throws Exceptio assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1)); } - public void innerTestParseWithProvidedPipeline() throws Exception { + public void testParseWithProvidedPipeline() throws Exception { int numDocs = randomIntBetween(1, 10); Map requestContent = new HashMap<>(); @@ -149,9 +149,9 @@ public void innerTestParseWithProvidedPipeline() throws Exception { doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); } else { - Integer value = randomIntBetween(1, 1000000); + int value = randomIntBetween(1, 1000000); doc.put(field.getFieldName(), value); - expectedDoc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), (long) value); } } else if (field == VERSION_TYPE) { String value = VersionType.toString(randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL, VersionType.EXTERNAL_GTE)); @@ -163,9 +163,9 @@ public void innerTestParseWithProvidedPipeline() throws Exception { doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); } else { - Integer value = randomIntBetween(1, 1000000); + int value = randomIntBetween(1, 1000000); doc.put(field.getFieldName(), value); - expectedDoc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), (long) value); } } else { if (randomBoolean()) { From ffebb0ac5ac30360e47ab5bdde1d5ba39be3e520 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 17:20:37 -0400 Subject: [PATCH 138/388] Bump commons-io:commons-io from 2.13.0 to 2.14.0 in /plugins/repository-hdfs (#10505) * Bump commons-io:commons-io in /plugins/repository-hdfs Bumps commons-io:commons-io from 2.13.0 to 2.14.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/commons-io-2.13.0.jar.sha1 | 1 - plugins/repository-hdfs/licenses/commons-io-2.14.0.jar.sha1 | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/commons-io-2.13.0.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-io-2.14.0.jar.sha1 diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 8d9ff01f5db65..1ce29afe0e69a 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -75,7 +75,7 @@ dependencies { api 'commons-collections:commons-collections:3.2.2' api "org.apache.commons:commons-compress:${versions.commonscompress}" api 'org.apache.commons:commons-configuration2:2.9.0' - api 'commons-io:commons-io:2.13.0' + api 'commons-io:commons-io:2.14.0' api 'org.apache.commons:commons-lang3:3.13.0' implementation 'com.google.re2j:re2j:1.7' api 'javax.servlet:servlet-api:2.5' diff --git a/plugins/repository-hdfs/licenses/commons-io-2.13.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-io-2.13.0.jar.sha1 deleted file mode 100644 index c165136eb5822..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-io-2.13.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8bb2bc9b4df17e2411533a0708a69f983bf5e83b \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-io-2.14.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-io-2.14.0.jar.sha1 new file mode 100644 index 0000000000000..33c5cfe53e01d --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-io-2.14.0.jar.sha1 @@ -0,0 +1 @@ +a4c6e1f6c196339473cd2e1b037f0eb97c62755b \ No newline at end of file From 332fae1ee113c16ca478d7a6ab47c897ae3139e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 17:23:44 -0400 Subject: [PATCH 139/388] Bump org.apache.zookeeper:zookeeper from 3.9.0 to 3.9.1 in /test/fixtures/hdfs-fixture (#10506) * Bump org.apache.zookeeper:zookeeper in /test/fixtures/hdfs-fixture Bumps org.apache.zookeeper:zookeeper from 3.9.0 to 3.9.1. --- updated-dependencies: - dependency-name: org.apache.zookeeper:zookeeper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 3 ++- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba71604d1a94e..05a3fa8cb785b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301)) - Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210)) - Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297)) +- Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506)) ### Changed - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415)) @@ -154,4 +155,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x \ No newline at end of file diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 7149efe27d694..7de98c0986b87 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -64,7 +64,7 @@ dependencies { api "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" api "org.eclipse.jetty:jetty-server:${versions.jetty}" api "org.eclipse.jetty.websocket:javax-websocket-server-impl:${versions.jetty}" - api 'org.apache.zookeeper:zookeeper:3.9.0' + api 'org.apache.zookeeper:zookeeper:3.9.1' api "org.apache.commons:commons-text:1.10.0" api "commons-net:commons-net:3.9.0" runtimeOnly "com.google.guava:guava:${versions.guava}" From 3c76cec7ec6c8c78e2a6884e976e7547f60757db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 17:24:36 -0400 Subject: [PATCH 140/388] Bump com.google.api:gax from 2.32.0 to 2.35.0 in /plugins/repository-gcs (#10507) * Bump com.google.api:gax from 2.32.0 to 2.35.0 in /plugins/repository-gcs Bumps com.google.api:gax from 2.32.0 to 2.35.0. --- updated-dependencies: - dependency-name: com.google.api:gax dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- plugins/repository-gcs/build.gradle | 2 +- plugins/repository-gcs/licenses/gax-2.32.0.jar.sha1 | 1 - plugins/repository-gcs/licenses/gax-2.35.0.jar.sha1 | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/gax-2.32.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/gax-2.35.0.jar.sha1 diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 52e5e71618d69..da4978608a12f 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -53,7 +53,7 @@ versions << [ dependencies { api 'com.google.api:api-common:1.8.1' - api 'com.google.api:gax:2.32.0' + api 'com.google.api:gax:2.35.0' api 'com.google.api:gax-httpjson:0.103.1' api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0' diff --git a/plugins/repository-gcs/licenses/gax-2.32.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-2.32.0.jar.sha1 deleted file mode 100644 index 9cae74e1c3673..0000000000000 --- a/plugins/repository-gcs/licenses/gax-2.32.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -522bf3c2a738847b9719eac8ce572be0f84da40a \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gax-2.35.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-2.35.0.jar.sha1 new file mode 100644 index 0000000000000..778922c637dc1 --- /dev/null +++ b/plugins/repository-gcs/licenses/gax-2.35.0.jar.sha1 @@ -0,0 +1 @@ +98d52034cfa6d1b881e16f418894afcfacd89b7a \ No newline at end of file From ef6dafe3948bf5984063927c2b14c9597d6d9df3 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 10 Oct 2023 14:30:25 -0700 Subject: [PATCH 141/388] add taskCompletionCount in search_backpressure stats (#10028) * add taskCompletionCount in search_backpressure Signed-off-by: Kaushal Kumar * address comments to use final objects Signed-off-by: Kaushal Kumar * do spotless check run Signed-off-by: Kaushal Kumar * use primitive long to store completionCount Signed-off-by: Kaushal Kumar * use primitive long to store completionCount in searchTaskStats Signed-off-by: Kaushal Kumar * add pr link against the change Signed-off-by: Kaushal Kumar * add taskCompletionCount in search_backpressure Signed-off-by: Kaushal Kumar * address comments to use final objects Signed-off-by: Kaushal Kumar * do spotless check run Signed-off-by: Kaushal Kumar * use primitive long to store completionCount Signed-off-by: Kaushal Kumar * rebase with upstream main Signed-off-by: Kaushal Kumar * rebase with upstream main Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kaushal Kumar --- CHANGELOG.md | 2 ++ .../SearchBackpressureService.java | 2 ++ .../stats/SearchShardTaskStats.java | 18 +++++++++++++++++- .../backpressure/stats/SearchTaskStats.java | 18 +++++++++++++++++- .../SearchBackpressureServiceTests.java | 14 ++++++++------ .../stats/SearchShardTaskStatsTests.java | 7 ++++++- .../stats/SearchTaskStatsTests.java | 2 +- 7 files changed, 53 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05a3fa8cb785b..467e019270a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,8 +54,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855)) +- Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/)) - Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558)) + ### Deprecated ### Removed diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index 4f6c2c327509d..29fbee416a9b8 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -399,6 +399,7 @@ public SearchBackpressureStats nodeStats() { SearchTaskStats searchTaskStats = new SearchTaskStats( searchBackpressureStates.get(SearchTask.class).getCancellationCount(), searchBackpressureStates.get(SearchTask.class).getLimitReachedCount(), + searchBackpressureStates.get(SearchTask.class).getCompletionCount(), taskTrackers.get(SearchTask.class) .stream() .collect(Collectors.toUnmodifiableMap(t -> TaskResourceUsageTrackerType.fromName(t.name()), t -> t.stats(searchTasks))) @@ -407,6 +408,7 @@ public SearchBackpressureStats nodeStats() { SearchShardTaskStats searchShardTaskStats = new SearchShardTaskStats( searchBackpressureStates.get(SearchShardTask.class).getCancellationCount(), searchBackpressureStates.get(SearchShardTask.class).getLimitReachedCount(), + searchBackpressureStates.get(SearchShardTask.class).getCompletionCount(), taskTrackers.get(SearchShardTask.class) .stream() .collect(Collectors.toUnmodifiableMap(t -> TaskResourceUsageTrackerType.fromName(t.name()), t -> t.stats(searchShardTasks))) diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java index 678c19d83fb96..ffe97d125b27a 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java +++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java @@ -8,6 +8,7 @@ package org.opensearch.search.backpressure.stats; +import org.opensearch.Version; import org.opensearch.common.collect.MapBuilder; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -30,21 +31,29 @@ public class SearchShardTaskStats implements ToXContentObject, Writeable { private final long cancellationCount; private final long limitReachedCount; + private final long completionCount; private final Map resourceUsageTrackerStats; public SearchShardTaskStats( long cancellationCount, long limitReachedCount, + long completionCount, Map resourceUsageTrackerStats ) { this.cancellationCount = cancellationCount; this.limitReachedCount = limitReachedCount; + this.completionCount = completionCount; this.resourceUsageTrackerStats = resourceUsageTrackerStats; } public SearchShardTaskStats(StreamInput in) throws IOException { this.cancellationCount = in.readVLong(); this.limitReachedCount = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + completionCount = in.readVLong(); + } else { + completionCount = -1; + } MapBuilder builder = new MapBuilder<>(); builder.put(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, in.readOptionalWriteable(CpuUsageTracker.Stats::new)); @@ -62,6 +71,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(entry.getKey().getName(), entry.getValue()); } builder.endObject(); + if (completionCount != -1) { + builder.field("completion_count", completionCount); + } builder.startObject("cancellation_stats") .field("cancellation_count", cancellationCount) @@ -75,6 +87,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public void writeTo(StreamOutput out) throws IOException { out.writeVLong(cancellationCount); out.writeVLong(limitReachedCount); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeVLong(completionCount); + } out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER)); out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER)); @@ -88,11 +103,12 @@ public boolean equals(Object o) { SearchShardTaskStats that = (SearchShardTaskStats) o; return cancellationCount == that.cancellationCount && limitReachedCount == that.limitReachedCount + && completionCount == that.completionCount && resourceUsageTrackerStats.equals(that.resourceUsageTrackerStats); } @Override public int hashCode() { - return Objects.hash(cancellationCount, limitReachedCount, resourceUsageTrackerStats); + return Objects.hash(cancellationCount, limitReachedCount, resourceUsageTrackerStats, completionCount); } } diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java index 302350104bd3a..a7f9b4e3d004f 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java +++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java @@ -8,6 +8,7 @@ package org.opensearch.search.backpressure.stats; +import org.opensearch.Version; import org.opensearch.common.collect.MapBuilder; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -31,21 +32,29 @@ public class SearchTaskStats implements ToXContentObject, Writeable { private final long cancellationCount; private final long limitReachedCount; + private final long completionCount; private final Map resourceUsageTrackerStats; public SearchTaskStats( long cancellationCount, long limitReachedCount, + long completionCount, Map resourceUsageTrackerStats ) { this.cancellationCount = cancellationCount; this.limitReachedCount = limitReachedCount; + this.completionCount = completionCount; this.resourceUsageTrackerStats = resourceUsageTrackerStats; } public SearchTaskStats(StreamInput in) throws IOException { this.cancellationCount = in.readVLong(); this.limitReachedCount = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.completionCount = in.readVLong(); + } else { + this.completionCount = -1; + } MapBuilder builder = new MapBuilder<>(); builder.put(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, in.readOptionalWriteable(CpuUsageTracker.Stats::new)); @@ -63,6 +72,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(entry.getKey().getName(), entry.getValue()); } builder.endObject(); + if (completionCount != -1) { + builder.field("completion_count", completionCount); + } builder.startObject("cancellation_stats") .field("cancellation_count", cancellationCount) @@ -76,6 +88,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public void writeTo(StreamOutput out) throws IOException { out.writeVLong(cancellationCount); out.writeVLong(limitReachedCount); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeVLong(completionCount); + } out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER)); out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER)); @@ -89,11 +104,12 @@ public boolean equals(Object o) { SearchTaskStats that = (SearchTaskStats) o; return cancellationCount == that.cancellationCount && limitReachedCount == that.limitReachedCount + && completionCount == that.completionCount && resourceUsageTrackerStats.equals(that.resourceUsageTrackerStats); } @Override public int hashCode() { - return Objects.hash(cancellationCount, limitReachedCount, resourceUsageTrackerStats); + return Objects.hash(cancellationCount, limitReachedCount, resourceUsageTrackerStats, completionCount); } } diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java index f0d930c4c3acb..9778798b706f4 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java @@ -249,10 +249,11 @@ public void testSearchTaskInFlightCancellation() { verify(mockTaskManager, times(10)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any()); assertEquals(3, service.getSearchBackpressureState(SearchTask.class).getLimitReachedCount()); - // Verify search backpressure stats. + // Verify search backpressure stats. Since we are not marking any task as completed the completionCount will be 0 + // for SearchTaskStats here. SearchBackpressureStats expectedStats = new SearchBackpressureStats( - new SearchTaskStats(10, 3, Map.of(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, new MockStats(10))), - new SearchShardTaskStats(0, 0, Collections.emptyMap()), + new SearchTaskStats(10, 3, 0, Map.of(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, new MockStats(10))), + new SearchShardTaskStats(0, 0, 0, Collections.emptyMap()), SearchBackpressureMode.ENFORCED ); SearchBackpressureStats actualStats = service.nodeStats(); @@ -323,10 +324,11 @@ public void testSearchShardTaskInFlightCancellation() { verify(mockTaskManager, times(12)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any()); assertEquals(3, service.getSearchBackpressureState(SearchShardTask.class).getLimitReachedCount()); - // Verify search backpressure stats. + // Verify search backpressure stats. We are marking 20 SearchShardTasks as completed this should get + // reflected in SearchShardTaskStats. SearchBackpressureStats expectedStats = new SearchBackpressureStats( - new SearchTaskStats(0, 0, Collections.emptyMap()), - new SearchShardTaskStats(12, 3, Map.of(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, new MockStats(12))), + new SearchTaskStats(0, 0, 0, Collections.emptyMap()), + new SearchShardTaskStats(12, 3, 20, Map.of(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, new MockStats(12))), SearchBackpressureMode.ENFORCED ); SearchBackpressureStats actualStats = service.nodeStats(); diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java index 6478fdfff61d4..f28b82cad30d3 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java @@ -39,6 +39,11 @@ public static SearchShardTaskStats randomInstance() { new ElapsedTimeTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) ); - return new SearchShardTaskStats(randomNonNegativeLong(), randomNonNegativeLong(), resourceUsageTrackerStats); + return new SearchShardTaskStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + resourceUsageTrackerStats + ); } } diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java index eb33bc1c37b7e..cc7aa92826b41 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java @@ -40,6 +40,6 @@ public static SearchTaskStats randomInstance() { new ElapsedTimeTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) ); - return new SearchTaskStats(randomNonNegativeLong(), randomNonNegativeLong(), resourceUsageTrackerStats); + return new SearchTaskStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), resourceUsageTrackerStats); } } From d3c8909f8c7b6c786af101ca0ab3dacaa84b79b1 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 10 Oct 2023 16:37:06 -0500 Subject: [PATCH 142/388] Mute MoreExpressionIT.testSpecialValueVariable (#10388) Mute the test only for the concurrent search enabled case. See #10079. Signed-off-by: Andrew Ross --- .../org/opensearch/script/expression/MoreExpressionIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java index 8ca28a905f216..bb2f652168d5c 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java @@ -504,6 +504,10 @@ public void testInvalidFieldMember() { } public void testSpecialValueVariable() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10079", + internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + ); // i.e. _value for aggregations createIndex("test"); ensureGreen("test"); From 00ccfc49f3aa2bf06ddc22ff8f0bc24790b222da Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 10 Oct 2023 19:54:15 -0500 Subject: [PATCH 143/388] Replace multipart download with parallel file download (#10519) There are a few open issues with the multi-stream download approach: - Recovery stats are not being reported correctly - It is incompatible (short of reopening and re-reading the entire file) with the existing Lucene checksum validation logic - There are some issues with integrating it with the pending client side encryption work Given this, I attempted an experiment where I replaced with multi-stream-within-a-single-file approach with simply parallelizing downloads across files (this is how snapshot restore works). I actually got better results with this approach: recovering a ~52GiB shard took about 4.7 minutes with the multi-stream code versus 3.9 minutes with the parallel file approach (r7g.4xlarge EC2 instance, 500MiB/s EBS volume, S3 as remote repository). I think this is the right approach as it leverages the more battle-tested code path and addresses the three issues listed above. The multi-stream approach still has promise as it will allow us to download very large files faster (whereas this approach they can be the long poll on the transfer operation). However, given that 5GB segments (made up of multiple files in practice) are the norm, we generally aren't dealing with huge files. Signed-off-by: Andrew Ross --- .../org/opensearch/index/IndexModule.java | 7 +- .../org/opensearch/index/IndexService.java | 9 +- .../opensearch/index/shard/IndexShard.java | 47 ++---- .../opensearch/index/shard/StoreRecovery.java | 11 +- .../index/store/RemoteDirectory.java | 4 - .../store/RemoteSegmentStoreDirectory.java | 74 +-------- .../RemoteSegmentStoreDirectoryFactory.java | 11 +- .../store/RemoteStoreFileDownloader.java | 147 ++++++++++++++++++ .../opensearch/indices/IndicesService.java | 9 +- .../indices/recovery/RecoverySettings.java | 4 +- .../RemoteStoreReplicationSource.java | 32 +--- .../main/java/org/opensearch/node/Node.java | 6 +- .../blobstore/BlobStoreRepository.java | 6 +- .../opensearch/index/IndexModuleTests.java | 5 +- .../RemoteStoreRefreshListenerTests.java | 4 +- ...moteSegmentStoreDirectoryFactoryTests.java | 7 +- .../RemoteSegmentStoreDirectoryTests.java | 127 +-------------- .../store/RemoteStoreFileDownloaderTests.java | 119 ++++++++++++++ .../snapshots/SnapshotResiliencyTests.java | 5 +- .../index/shard/IndexShardTestCase.java | 11 +- 20 files changed, 333 insertions(+), 312 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java create mode 100644 server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 8692876412ea9..e29283724ebf8 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -81,6 +81,7 @@ import org.opensearch.indices.IndicesQueryCache; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; @@ -602,7 +603,8 @@ public IndexService newIndexService( IndexStorePlugin.DirectoryFactory remoteDirectoryFactory, BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, - Supplier clusterRemoteTranslogBufferIntervalSupplier + Supplier clusterRemoteTranslogBufferIntervalSupplier, + RecoverySettings recoverySettings ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -660,7 +662,8 @@ public IndexService newIndexService( recoveryStateFactory, translogFactorySupplier, clusterDefaultRefreshIntervalSupplier, - clusterRemoteTranslogBufferIntervalSupplier + clusterRemoteTranslogBufferIntervalSupplier, + recoverySettings ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index af23145be9f89..a53dbe246fa44 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -89,13 +89,13 @@ import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.similarity.SimilarityService; -import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.plugins.IndexStorePlugin; @@ -179,6 +179,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final BiFunction translogFactorySupplier; private final Supplier clusterDefaultRefreshIntervalSupplier; private final Supplier clusterRemoteTranslogBufferIntervalSupplier; + private final RecoverySettings recoverySettings; public IndexService( IndexSettings indexSettings, @@ -213,7 +214,8 @@ public IndexService( IndexStorePlugin.RecoveryStateFactory recoveryStateFactory, BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, - Supplier clusterRemoteTranslogBufferIntervalSupplier + Supplier clusterRemoteTranslogBufferIntervalSupplier, + RecoverySettings recoverySettings ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -290,6 +292,7 @@ public IndexService( this.retentionLeaseSyncTask = new AsyncRetentionLeaseSyncTask(this); this.translogFactorySupplier = translogFactorySupplier; this.clusterRemoteTranslogBufferIntervalSupplier = clusterRemoteTranslogBufferIntervalSupplier; + this.recoverySettings = recoverySettings; updateFsyncTaskIfNecessary(); } @@ -522,7 +525,7 @@ public synchronized IndexShard createShard( remoteStoreStatsTrackerFactory, clusterRemoteTranslogBufferIntervalSupplier, nodeEnv.nodeId(), - (RemoteSegmentStoreDirectoryFactory) remoteDirectoryFactory + recoverySettings ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 251f9a5ae01c0..2643af501bbc9 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -62,7 +62,6 @@ import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.opensearch.action.admin.indices.upgrade.post.UpgradeRequest; -import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.replication.PendingReplicationActions; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.cluster.metadata.DataStream; @@ -160,9 +159,8 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; -import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; -import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; +import org.opensearch.index.store.RemoteStoreFileDownloader; import org.opensearch.index.store.Store; import org.opensearch.index.store.Store.MetadataSnapshot; import org.opensearch.index.store.StoreFileMetadata; @@ -184,6 +182,7 @@ import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoveryFailedException; import org.opensearch.indices.recovery.RecoveryListener; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.recovery.RecoveryTarget; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; @@ -201,7 +200,6 @@ import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.NoSuchFileException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -342,7 +340,7 @@ Runnable getGlobalCheckpointSyncer() { private final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory; private final List internalRefreshListener = new ArrayList<>(); - private final RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory; + private final RemoteStoreFileDownloader fileDownloader; public IndexShard( final ShardRouting shardRouting, @@ -371,10 +369,7 @@ public IndexShard( final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, final Supplier clusterRemoteTranslogBufferIntervalSupplier, final String nodeId, - // Wiring a directory factory here breaks some intended abstractions, but this remote directory - // factory is used not as a Lucene directory but instead to copy files from a remote store when - // restoring a shallow snapshot. - @Nullable final RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory + final RecoverySettings recoverySettings ) throws IOException { super(shardRouting.shardId(), indexSettings); assert shardRouting.initializing(); @@ -470,7 +465,7 @@ public boolean shouldCache(Query query) { ? false : mapperService.documentMapper().mappers().containsTimeStampField(); this.remoteStoreStatsTrackerFactory = remoteStoreStatsTrackerFactory; - this.remoteSegmentStoreDirectoryFactory = remoteSegmentStoreDirectoryFactory; + this.fileDownloader = new RemoteStoreFileDownloader(shardRouting.shardId(), threadPool, recoverySettings); } public ThreadPool getThreadPool() { @@ -568,6 +563,10 @@ public String getNodeId() { return translogConfig.getNodeId(); } + public RemoteStoreFileDownloader getFileDownloader() { + return fileDownloader; + } + @Override public void updateShardState( final ShardRouting newRouting, @@ -2706,7 +2705,7 @@ public void restoreFromRemoteStore(ActionListener listener) { public void restoreFromSnapshotAndRemoteStore( Repository repository, - RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + RepositoriesService repositoriesService, ActionListener listener ) { try { @@ -2714,7 +2713,7 @@ public void restoreFromSnapshotAndRemoteStore( assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + recoveryState.getRecoverySource(); StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); - storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, remoteSegmentStoreDirectoryFactory, listener); + storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, repositoriesService, listener, threadPool); } catch (Exception e) { listener.onFailure(e); } @@ -3554,7 +3553,7 @@ public void startRecovery( "from snapshot and remote store", recoveryState, recoveryListener, - l -> restoreFromSnapshotAndRemoteStore(repositoriesService.repository(repo), remoteSegmentStoreDirectoryFactory, l) + l -> restoreFromSnapshotAndRemoteStore(repositoriesService.repository(repo), repositoriesService, l) ); // indicesService.indexService(shardRouting.shardId().getIndex()).addMetadataListener(); } else { @@ -4912,7 +4911,7 @@ private String copySegmentFiles( if (toDownloadSegments.isEmpty() == false) { try { - downloadSegments(storeDirectory, sourceRemoteDirectory, targetRemoteDirectory, toDownloadSegments, onFileSync); + fileDownloader.download(sourceRemoteDirectory, storeDirectory, targetRemoteDirectory, toDownloadSegments, onFileSync); } catch (Exception e) { throw new IOException("Error occurred when downloading segments from remote store", e); } @@ -4925,26 +4924,6 @@ private String copySegmentFiles( return segmentNFile; } - private void downloadSegments( - Directory storeDirectory, - RemoteSegmentStoreDirectory sourceRemoteDirectory, - RemoteSegmentStoreDirectory targetRemoteDirectory, - Set toDownloadSegments, - final Runnable onFileSync - ) throws IOException { - final Path indexPath = store.shardPath() == null ? null : store.shardPath().resolveIndex(); - final DirectoryFileTransferTracker fileTransferTracker = store.getDirectoryFileTransferTracker(); - for (String segment : toDownloadSegments) { - final PlainActionFuture segmentListener = PlainActionFuture.newFuture(); - sourceRemoteDirectory.copyTo(segment, storeDirectory, indexPath, fileTransferTracker, segmentListener); - segmentListener.actionGet(); - onFileSync.run(); - if (targetRemoteDirectory != null) { - targetRemoteDirectory.copyFrom(storeDirectory, segment, segment, IOContext.DEFAULT); - } - } - } - private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) { try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) { if (checksum == CodecUtil.retrieveChecksum(indexInput)) { diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 762aab51469d0..c0211e1257c8e 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -70,7 +70,9 @@ import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.repositories.IndexId; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.nio.channels.FileChannel; @@ -360,8 +362,9 @@ void recoverFromRepository(final IndexShard indexShard, Repository repository, A void recoverFromSnapshotAndRemoteStore( final IndexShard indexShard, Repository repository, - RemoteSegmentStoreDirectoryFactory directoryFactory, - ActionListener listener + RepositoriesService repositoriesService, + ActionListener listener, + ThreadPool threadPool ) { try { if (canRecover(indexShard)) { @@ -389,6 +392,10 @@ void recoverFromSnapshotAndRemoteStore( remoteStoreRepository = shallowCopyShardMetadata.getRemoteStoreRepository(); } + RemoteSegmentStoreDirectoryFactory directoryFactory = new RemoteSegmentStoreDirectoryFactory( + () -> repositoriesService, + threadPool + ); RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( remoteStoreRepository, indexUUID, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index 70a88f6159764..345583bbbd1be 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -336,10 +336,6 @@ public boolean copyFrom( return false; } - protected UnaryOperator getDownloadRateLimiter() { - return downloadRateLimiter; - } - private void uploadBlob( Directory from, String src, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index c0b302c68c0ea..be1f2341236ab 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -24,8 +24,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Version; import org.opensearch.common.UUIDs; -import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; -import org.opensearch.common.blobstore.stream.read.listener.ReadContextListener; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.logging.Loggers; @@ -38,7 +36,6 @@ import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; -import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.threadpool.ThreadPool; @@ -46,7 +43,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -93,8 +89,6 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory implement private final ThreadPool threadPool; - private final RecoverySettings recoverySettings; - /** * Keeps track of local segment filename to uploaded filename along with other attributes like checksum. * This map acts as a cache layer for uploaded segment filenames which helps avoid calling listAll() each time. @@ -127,15 +121,13 @@ public RemoteSegmentStoreDirectory( RemoteDirectory remoteMetadataDirectory, RemoteStoreLockManager mdLockManager, ThreadPool threadPool, - ShardId shardId, - RecoverySettings recoverySettings + ShardId shardId ) throws IOException { super(remoteDataDirectory); this.remoteDataDirectory = remoteDataDirectory; this.remoteMetadataDirectory = remoteMetadataDirectory; this.mdLockManager = mdLockManager; this.threadPool = threadPool; - this.recoverySettings = recoverySettings; this.logger = Loggers.getLogger(getClass(), shardId); init(); } @@ -473,70 +465,6 @@ public void copyFrom(Directory from, String src, IOContext context, ActionListen } } - /** - * Copies an existing {@code source} file from this directory to a non-existent file (also - * named {@code source}) in either {@code destinationDirectory} or {@code destinationPath}. - * If the blob container backing this directory supports multipart downloads, the {@code source} - * file will be downloaded (potentially in multiple concurrent parts) directly to - * {@code destinationPath}. This method will return immediately and {@code fileCompletionListener} - * will be notified upon completion. - *

- * If multipart downloads are not supported, then {@code source} file will be copied to a file named - * {@code source} in a single part to {@code destinationDirectory}. The download will happen on the - * calling thread and {@code fileCompletionListener} will be notified synchronously before this - * method returns. - * - * @param source The source file name - * @param destinationDirectory The destination directory (if multipart is not supported) - * @param destinationPath The destination path (if multipart is supported) - * @param fileTransferTracker Tracker used for file transfer stats - * @param fileCompletionListener The listener to notify of completion - */ - public void copyTo( - String source, - Directory destinationDirectory, - Path destinationPath, - DirectoryFileTransferTracker fileTransferTracker, - ActionListener fileCompletionListener - ) { - final String blobName = getExistingRemoteFilename(source); - if (destinationPath != null && remoteDataDirectory.getBlobContainer() instanceof AsyncMultiStreamBlobContainer) { - long length = 0L; - try { - length = fileLength(source); - } catch (IOException ex) { - logger.error("Unable to fetch segment length for stats tracking", ex); - } - final long fileLength = length; - final long startTime = System.currentTimeMillis(); - fileTransferTracker.addTransferredBytesStarted(fileLength); - final AsyncMultiStreamBlobContainer blobContainer = (AsyncMultiStreamBlobContainer) remoteDataDirectory.getBlobContainer(); - final Path destinationFilePath = destinationPath.resolve(source); - final ActionListener completionListener = ActionListener.wrap(response -> { - fileTransferTracker.addTransferredBytesSucceeded(fileLength, startTime); - fileCompletionListener.onResponse(response); - }, e -> { - fileTransferTracker.addTransferredBytesFailed(fileLength, startTime); - fileCompletionListener.onFailure(e); - }); - final ReadContextListener readContextListener = new ReadContextListener( - blobName, - destinationFilePath, - completionListener, - threadPool, - remoteDataDirectory.getDownloadRateLimiter(), - recoverySettings.getMaxConcurrentRemoteStoreStreams() - ); - blobContainer.readBlobAsync(blobName, readContextListener); - } else { - // Fallback to older mechanism of downloading the file - ActionListener.completeWith(fileCompletionListener, () -> { - destinationDirectory.copyFrom(this, source, source, IOContext.DEFAULT); - return source; - }); - } - } - /** * This acquires a lock on a given commit by creating a lock file in lock directory using {@code FileLockInfo} * diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index cc55380894ecd..a5e89ec6a8327 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -15,7 +15,6 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; -import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.plugins.IndexStorePlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -35,18 +34,12 @@ public class RemoteSegmentStoreDirectoryFactory implements IndexStorePlugin.Dire private static final String SEGMENTS = "segments"; private final Supplier repositoriesService; - private final RecoverySettings recoverySettings; private final ThreadPool threadPool; - public RemoteSegmentStoreDirectoryFactory( - Supplier repositoriesService, - ThreadPool threadPool, - RecoverySettings recoverySettings - ) { + public RemoteSegmentStoreDirectoryFactory(Supplier repositoriesService, ThreadPool threadPool) { this.repositoriesService = repositoriesService; this.threadPool = threadPool; - this.recoverySettings = recoverySettings; } @Override @@ -78,7 +71,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s String.valueOf(shardId.id()) ); - return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool, shardId, recoverySettings); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool, shardId); } catch (RepositoryMissingException e) { throw new IllegalArgumentException("Repository should be created before creating index with remote_store enabled setting", e); } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java b/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java new file mode 100644 index 0000000000000..4fc721f2b96b5 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.InternalApi; +import org.opensearch.common.logging.Loggers; +import org.opensearch.common.util.concurrent.UncategorizedExecutionException; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.Collection; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; + +/** + * Helper class to downloads files from a {@link RemoteSegmentStoreDirectory} + * instance to a local {@link Directory} instance in parallel depending on thread + * pool size and recovery settings. + */ +@InternalApi +public final class RemoteStoreFileDownloader { + private final Logger logger; + private final ThreadPool threadPool; + private final RecoverySettings recoverySettings; + + public RemoteStoreFileDownloader(ShardId shardId, ThreadPool threadPool, RecoverySettings recoverySettings) { + this.logger = Loggers.getLogger(RemoteStoreFileDownloader.class, shardId); + this.threadPool = threadPool; + this.recoverySettings = recoverySettings; + } + + /** + * Copies the given segments from the remote segment store to the given + * local directory. + * @param source The remote directory to copy segment files from + * @param destination The local directory to copy segment files to + * @param toDownloadSegments The list of segment files to download + */ + public void download(Directory source, Directory destination, Collection toDownloadSegments) throws IOException { + downloadInternal(source, destination, null, toDownloadSegments, () -> {}); + } + + /** + * Copies the given segments from the remote segment store to the given + * local directory, while also copying the segments _to_ another remote directory. + * @param source The remote directory to copy segment files from + * @param destination The local directory to copy segment files to + * @param secondDestination The second remote directory that segment files are + * copied to after being copied to the local directory + * @param toDownloadSegments The list of segment files to download + * @param onFileCompletion A generic runnable that is invoked after each file download. + * Must be thread safe as this may be invoked concurrently from + * different threads. + */ + public void download( + Directory source, + Directory destination, + Directory secondDestination, + Collection toDownloadSegments, + Runnable onFileCompletion + ) throws IOException { + downloadInternal(source, destination, secondDestination, toDownloadSegments, onFileCompletion); + } + + private void downloadInternal( + Directory source, + Directory destination, + @Nullable Directory secondDestination, + Collection toDownloadSegments, + Runnable onFileCompletion + ) throws IOException { + final Queue queue = new ConcurrentLinkedQueue<>(toDownloadSegments); + // Choose the minimum of: + // - number of files to download + // - max thread pool size + // - "indices.recovery.max_concurrent_remote_store_streams" setting + final int threads = Math.min( + toDownloadSegments.size(), + Math.min(threadPool.info(ThreadPool.Names.REMOTE_RECOVERY).getMax(), recoverySettings.getMaxConcurrentRemoteStoreStreams()) + ); + logger.trace("Starting download of {} files with {} threads", queue.size(), threads); + final PlainActionFuture> listener = PlainActionFuture.newFuture(); + final ActionListener allFilesListener = new GroupedActionListener<>(listener, threads); + for (int i = 0; i < threads; i++) { + copyOneFile(source, destination, secondDestination, queue, onFileCompletion, allFilesListener); + } + try { + listener.actionGet(); + } catch (UncategorizedExecutionException e) { + // Any IOException will be double-wrapped so dig it out and throw it + if (e.getCause() instanceof ExecutionException) { + if (e.getCause().getCause() instanceof IOException) { + throw (IOException) e.getCause().getCause(); + } + } + throw e; + } + } + + private void copyOneFile( + Directory source, + Directory destination, + @Nullable Directory secondDestination, + Queue queue, + Runnable onFileCompletion, + ActionListener listener + ) { + final String file = queue.poll(); + if (file == null) { + // Queue is empty, so notify listener we are done + listener.onResponse(null); + } else { + threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY).submit(() -> { + logger.trace("Downloading file {}", file); + try { + destination.copyFrom(source, file, file, IOContext.DEFAULT); + onFileCompletion.run(); + if (secondDestination != null) { + secondDestination.copyFrom(destination, file, file, IOContext.DEFAULT); + } + } catch (Exception e) { + // Clear the queue to stop any future processing, report the failure, then return + queue.clear(); + listener.onFailure(e); + return; + } + copyOneFile(source, destination, secondDestination, queue, onFileCompletion, listener); + }); + } + } +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index a72142e65c5e8..e27a597007b62 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -144,6 +144,7 @@ import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoveryListener; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.indices.replication.common.ReplicationType; @@ -335,6 +336,7 @@ public class IndicesService extends AbstractLifecycleComponent private final CountDownLatch closeLatch = new CountDownLatch(1); private volatile boolean idFieldDataEnabled; private volatile boolean allowExpensiveQueries; + private final RecoverySettings recoverySettings; @Nullable private final OpenSearchThreadPoolExecutor danglingIndicesThreadPoolExecutor; @@ -380,7 +382,8 @@ public IndicesService( Supplier repositoriesServiceSupplier, FileCacheCleaner fileCacheCleaner, SearchRequestStats searchRequestStats, - @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory + @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, + RecoverySettings recoverySettings ) { this.settings = settings; this.threadPool = threadPool; @@ -477,6 +480,7 @@ protected void closeInternal() { this.clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(clusterService.getSettings()); clusterService.getClusterSettings() .addSettingsUpdateConsumer(CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, this::setClusterRemoteTranslogBufferInterval); + this.recoverySettings = recoverySettings; } /** @@ -874,7 +878,8 @@ private synchronized IndexService createIndexService( remoteDirectoryFactory, translogFactorySupplier, this::getClusterDefaultRefreshInterval, - this::getClusterRemoteTranslogBufferInterval + this::getClusterRemoteTranslogBufferInterval, + this.recoverySettings ); } diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java index ed9755bf824ea..44dfb2f4cb00a 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java @@ -85,11 +85,11 @@ public class RecoverySettings { ); /** - * Controls the maximum number of streams that can be started concurrently when downloading from the remote store. + * Controls the maximum number of streams that can be started concurrently per recovery when downloading from the remote store. */ public static final Setting INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING = Setting.intSetting( "indices.recovery.max_concurrent_remote_store_streams", - 20, + 10, 1, Property.Dynamic, Property.NodeScope diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java index ddbcb86269aa9..d2000a56401f5 100644 --- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java @@ -14,20 +14,16 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.util.Version; -import org.opensearch.action.support.PlainActionFuture; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.core.action.ActionListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; -import org.opensearch.index.shard.ShardPath; -import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -109,49 +105,31 @@ public void getSegmentFiles( logger.debug("Downloading segment files from remote store {}", filesToFetch); RemoteSegmentMetadata remoteSegmentMetadata = remoteDirectory.readLatestMetadataFile(); - List toDownloadSegments = new ArrayList<>(); Collection directoryFiles = List.of(indexShard.store().directory().listAll()); if (remoteSegmentMetadata != null) { try { indexShard.store().incRef(); indexShard.remoteStore().incRef(); final Directory storeDirectory = indexShard.store().directory(); - final ShardPath shardPath = indexShard.shardPath(); + final List toDownloadSegmentNames = new ArrayList<>(); for (StoreFileMetadata fileMetadata : filesToFetch) { String file = fileMetadata.name(); assert directoryFiles.contains(file) == false : "Local store already contains the file " + file; - toDownloadSegments.add(fileMetadata); + toDownloadSegmentNames.add(file); } - final DirectoryFileTransferTracker fileTransferTracker = indexShard.store().getDirectoryFileTransferTracker(); - downloadSegments(storeDirectory, remoteDirectory, toDownloadSegments, shardPath, fileTransferTracker, listener); - logger.debug("Downloaded segment files from remote store {}", toDownloadSegments); + indexShard.getFileDownloader().download(remoteDirectory, storeDirectory, toDownloadSegmentNames); + logger.debug("Downloaded segment files from remote store {}", filesToFetch); } finally { indexShard.store().decRef(); indexShard.remoteStore().decRef(); } } + listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); } catch (Exception e) { listener.onFailure(e); } } - private void downloadSegments( - Directory storeDirectory, - RemoteSegmentStoreDirectory remoteStoreDirectory, - List toDownloadSegments, - ShardPath shardPath, - DirectoryFileTransferTracker fileTransferTracker, - ActionListener completionListener - ) { - final Path indexPath = shardPath == null ? null : shardPath.resolveIndex(); - for (StoreFileMetadata storeFileMetadata : toDownloadSegments) { - final PlainActionFuture segmentListener = PlainActionFuture.newFuture(); - remoteStoreDirectory.copyTo(storeFileMetadata.name(), storeDirectory, indexPath, fileTransferTracker, segmentListener); - segmentListener.actionGet(); - } - completionListener.onResponse(new GetSegmentFilesResponse(toDownloadSegments)); - } - @Override public String getDescription() { return "RemoteStoreReplicationSource"; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index f44c8c8bea7f1..5b3b064a47c66 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -771,8 +771,7 @@ protected Node( final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( repositoriesServiceReference::get, - threadPool, - recoverySettings + threadPool ); final SearchRequestStats searchRequestStats = new SearchRequestStats(); @@ -803,7 +802,8 @@ protected Node( repositoriesServiceReference::get, fileCacheCleaner, searchRequestStats, - remoteStoreStatsTrackerFactory + remoteStoreStatsTrackerFactory, + recoverySettings ); final AliasValidator aliasValidator = new AliasValidator(); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index bf85e19493203..8e6649c366eeb 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -1144,8 +1144,7 @@ private void executeStaleShardDelete( // see https://github.com/opensearch-project/OpenSearch/issues/8469 new RemoteSegmentStoreDirectoryFactory( remoteStoreLockManagerFactory.getRepositoriesService(), - threadPool, - recoverySettings + threadPool ).newDirectory( remoteStoreRepoForIndex, indexUUID, @@ -1615,8 +1614,7 @@ private void executeOneStaleIndexDelete( // see https://github.com/opensearch-project/OpenSearch/issues/8469 new RemoteSegmentStoreDirectoryFactory( remoteStoreLockManagerFactory.getRepositoriesService(), - threadPool, - recoverySettings + threadPool ).newDirectory( remoteStoreRepoForIndex, indexUUID, diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index bbd73bcf97aab..97bc822be7d51 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -258,10 +258,11 @@ private IndexService newIndexService(IndexModule module) throws IOException { writableRegistry(), () -> false, null, - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool, DefaultRecoverySettings.INSTANCE), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, - () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, + DefaultRecoverySettings.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 941f2f48e71af..5a13f57db2c87 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -33,7 +33,6 @@ import org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils; import org.opensearch.index.store.Store; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; -import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.threadpool.ThreadPool; @@ -156,8 +155,7 @@ public void testRemoteDirectoryInitThrowsException() throws IOException { remoteMetadataDirectory, mock(RemoteStoreLockManager.class), mock(ThreadPool.class), - shardId, - DefaultRecoverySettings.INSTANCE + shardId ); FilterDirectory remoteStoreFilterDirectory = new RemoteStoreRefreshListenerTests.TestFilterDirectory( new RemoteStoreRefreshListenerTests.TestFilterDirectory(remoteSegmentStoreDirectory) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index 78c7fe64cebd9..cad5e47531cc6 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -20,7 +20,6 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; -import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -58,11 +57,7 @@ public void setup() { repositoriesService = mock(RepositoriesService.class); threadPool = mock(ThreadPool.class); when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); - remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( - repositoriesServiceSupplier, - threadPool, - DefaultRecoverySettings.INSTANCE - ); + remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(repositoriesServiceSupplier, threadPool); } public void testNewDirectory() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 3e45699ef36eb..36cfd84ff960a 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -25,9 +25,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; -import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.common.blobstore.stream.write.WriteContext; -import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; @@ -42,7 +40,6 @@ import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; -import org.opensearch.indices.recovery.DefaultRecoverySettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.threadpool.ThreadPool; import org.junit.After; @@ -51,18 +48,15 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.NoSuchFileException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import java.util.function.UnaryOperator; import org.mockito.Mockito; @@ -72,8 +66,6 @@ import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata; import static org.hamcrest.CoreMatchers.is; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.contains; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -151,14 +143,12 @@ public void setup() throws IOException { remoteMetadataDirectory, mdLockManager, threadPool, - indexShard.shardId(), - DefaultRecoverySettings.INSTANCE + indexShard.shardId() ); try (Store store = indexShard.store()) { segmentInfos = store.readLastCommittedSegmentsInfo(); } - when(remoteDataDirectory.getDownloadRateLimiter()).thenReturn(UnaryOperator.identity()); when(threadPool.executor(ThreadPool.Names.REMOTE_PURGE)).thenReturn(executorService); when(threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY)).thenReturn(executorService); } @@ -568,118 +558,6 @@ public void onFailure(Exception e) {} storeDirectory.close(); } - public void testCopyFilesToMultipart() throws Exception { - String filename = "_0.cfe"; - populateMetadata(); - remoteSegmentStoreDirectory.init(); - - Directory storeDirectory = mock(Directory.class); - AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); - when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); - - Mockito.doAnswer(invocation -> { - ActionListener completionListener = invocation.getArgument(1); - final CompletableFuture future = new CompletableFuture<>(); - future.complete(new InputStreamContainer(new ByteArrayInputStream(new byte[] { 42 }), 0, 1)); - completionListener.onResponse(new ReadContext(1, List.of(() -> future), "")); - return null; - }).when(blobContainer).readBlobAsync(any(), any()); - - CountDownLatch downloadLatch = new CountDownLatch(1); - ActionListener completionListener = new ActionListener<>() { - @Override - public void onResponse(String unused) { - downloadLatch.countDown(); - } - - @Override - public void onFailure(Exception e) {} - }; - Path path = createTempDir(); - DirectoryFileTransferTracker directoryFileTransferTracker = new DirectoryFileTransferTracker(); - long sourceFileLengthInBytes = remoteSegmentStoreDirectory.fileLength(filename); - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, directoryFileTransferTracker, completionListener); - assertTrue(downloadLatch.await(5000, TimeUnit.SECONDS)); - verify(blobContainer, times(1)).readBlobAsync(contains(filename), any()); - verify(storeDirectory, times(0)).copyFrom(any(), any(), any(), any()); - - // Verify stats are updated to DirectoryFileTransferTracker - assertEquals(sourceFileLengthInBytes, directoryFileTransferTracker.getTransferredBytesSucceeded()); - } - - public void testCopyFilesTo() throws Exception { - String filename = "_0.cfe"; - populateMetadata(); - remoteSegmentStoreDirectory.init(); - - Directory storeDirectory = mock(Directory.class); - CountDownLatch downloadLatch = new CountDownLatch(1); - ActionListener completionListener = new ActionListener<>() { - @Override - public void onResponse(String unused) { - downloadLatch.countDown(); - } - - @Override - public void onFailure(Exception e) {} - }; - Path path = createTempDir(); - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, new DirectoryFileTransferTracker(), completionListener); - assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); - verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); - } - - public void testCopyFilesToEmptyPath() throws Exception { - String filename = "_0.cfe"; - populateMetadata(); - remoteSegmentStoreDirectory.init(); - - Directory storeDirectory = mock(Directory.class); - AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); - when(remoteDataDirectory.getBlobContainer()).thenReturn(blobContainer); - - CountDownLatch downloadLatch = new CountDownLatch(1); - ActionListener completionListener = new ActionListener<>() { - @Override - public void onResponse(String unused) { - downloadLatch.countDown(); - } - - @Override - public void onFailure(Exception e) {} - }; - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, null, new DirectoryFileTransferTracker(), completionListener); - assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); - verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); - } - - public void testCopyFilesToException() throws Exception { - String filename = "_0.cfe"; - populateMetadata(); - remoteSegmentStoreDirectory.init(); - - Directory storeDirectory = mock(Directory.class); - Mockito.doThrow(new IOException()) - .when(storeDirectory) - .copyFrom(any(Directory.class), anyString(), anyString(), any(IOContext.class)); - CountDownLatch downloadLatch = new CountDownLatch(1); - ActionListener completionListener = new ActionListener<>() { - @Override - public void onResponse(String unused) { - - } - - @Override - public void onFailure(Exception e) { - downloadLatch.countDown(); - } - }; - Path path = createTempDir(); - remoteSegmentStoreDirectory.copyTo(filename, storeDirectory, path, new DirectoryFileTransferTracker(), completionListener); - assertTrue(downloadLatch.await(5000, TimeUnit.MILLISECONDS)); - verify(storeDirectory, times(1)).copyFrom(any(), eq(filename), eq(filename), eq(IOContext.DEFAULT)); - } - public void testCopyFilesFromMultipartIOException() throws Exception { String filename = "_100.si"; AsyncMultiStreamBlobContainer blobContainer = mock(AsyncMultiStreamBlobContainer.class); @@ -689,8 +567,7 @@ public void testCopyFilesFromMultipartIOException() throws Exception { remoteMetadataDirectory, mdLockManager, threadPool, - indexShard.shardId(), - DefaultRecoverySettings.INSTANCE + indexShard.shardId() ); populateMetadata(); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java b/server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java new file mode 100644 index 0000000000000..588d9e8bb13a2 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.NIOFSDirectory; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class RemoteStoreFileDownloaderTests extends OpenSearchTestCase { + + private ThreadPool threadPool; + private Directory source; + private Directory destination; + private Directory secondDestination; + private RemoteStoreFileDownloader fileDownloader; + private Map files = new HashMap<>(); + + @Before + public void setup() throws IOException { + final int streamLimit = randomIntBetween(1, 20); + final RecoverySettings recoverySettings = new RecoverySettings( + Settings.builder().put("indices.recovery.max_concurrent_remote_store_streams", streamLimit).build(), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + threadPool = new TestThreadPool(getTestName()); + source = new NIOFSDirectory(createTempDir()); + destination = new NIOFSDirectory(createTempDir()); + secondDestination = new NIOFSDirectory(createTempDir()); + for (int i = 0; i < 10; i++) { + final String filename = "file_" + i; + final int content = randomInt(); + try (IndexOutput output = source.createOutput(filename, IOContext.DEFAULT)) { + output.writeInt(content); + } + files.put(filename, content); + } + fileDownloader = new RemoteStoreFileDownloader( + ShardId.fromString("[RemoteStoreFileDownloaderTests][0]"), + threadPool, + recoverySettings + ); + } + + @After + public void stopThreadPool() throws Exception { + threadPool.shutdown(); + assertTrue(threadPool.awaitTermination(5, TimeUnit.SECONDS)); + } + + public void testDownload() throws IOException { + fileDownloader.download(source, destination, files.keySet()); + assertContent(files, destination); + } + + public void testDownloadWithSecondDestination() throws IOException { + fileDownloader.download(source, destination, secondDestination, files.keySet(), () -> {}); + assertContent(files, destination); + assertContent(files, secondDestination); + } + + public void testDownloadWithFileCompletionHandler() throws IOException { + final AtomicInteger counter = new AtomicInteger(0); + fileDownloader.download(source, destination, null, files.keySet(), counter::incrementAndGet); + assertContent(files, destination); + assertEquals(files.size(), counter.get()); + } + + public void testDownloadNonExistentFile() { + assertThrows(NoSuchFileException.class, () -> fileDownloader.download(source, destination, Set.of("not real"))); + } + + public void testDownloadExtraNonExistentFile() { + List filesWithExtra = new ArrayList<>(files.keySet()); + filesWithExtra.add("not real"); + assertThrows(NoSuchFileException.class, () -> fileDownloader.download(source, destination, filesWithExtra)); + } + + private static void assertContent(Map expected, Directory destination) throws IOException { + // Note that Lucene will randomly write extra files (see org.apache.lucene.tests.mockfile.ExtraFS) + // so we just need to check that all the expected files are present but not that _only_ the expected + // files are present + final Set actualFiles = Set.of(destination.listAll()); + for (String file : expected.keySet()) { + assertTrue(actualFiles.contains(file)); + try (IndexInput input = destination.openInput(file, IOContext.DEFAULT)) { + assertEquals(expected.get(file), Integer.valueOf(input.readInt())); + assertThrows(EOFException.class, input::readByte); + } + } + } +} diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 80731b378f369..97c5d23831965 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2067,11 +2067,12 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool, DefaultRecoverySettings.INSTANCE), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), repositoriesServiceReference::get, fileCacheCleaner, null, - new RemoteStoreStatsTrackerFactory(clusterService, settings) + new RemoteStoreStatsTrackerFactory(clusterService, settings), + DefaultRecoverySettings.INSTANCE ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 186c1c7e78f6b..ffa37817e8a03 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -702,7 +702,7 @@ protected IndexShard newShard( remoteStoreStatsTrackerFactory, () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, "dummy-node", - null + DefaultRecoverySettings.INSTANCE ); indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER); if (remoteStoreStatsTrackerFactory != null) { @@ -789,14 +789,7 @@ protected RemoteSegmentStoreDirectory createRemoteSegmentStoreDirectory(ShardId RemoteStoreLockManager remoteStoreLockManager = new RemoteStoreMetadataLockManager( new RemoteBufferedOutputDirectory(getBlobContainer(remoteShardPath.resolveIndex())) ); - return new RemoteSegmentStoreDirectory( - dataDirectory, - metadataDirectory, - remoteStoreLockManager, - threadPool, - shardId, - DefaultRecoverySettings.INSTANCE - ); + return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, remoteStoreLockManager, threadPool, shardId); } private RemoteDirectory newRemoteDirectory(Path f) throws IOException { From c0852f84f0647f78a45e71396d6370a39ba4922f Mon Sep 17 00:00:00 2001 From: Laurent Laborde Date: Wed, 11 Oct 2023 04:42:47 +0200 Subject: [PATCH 144/388] New javadoc and improvement of existing javadoc (#10064) * Fixing javadoc error fixing javadoc error and warning javadoc for org.opensearch.core.indices.breaker javadoc for org.opensearch.core.index.shard.ShardId javadoc for org.opensearch.core.index.Index fixing a mishap in formatting rule fixing javadoc of org.opensearch.cli.Terminal Signed-off-by: Laurent Laborde * removing unsupported @ImplNote Signed-off-by: Laurent Laborde * Update libs/common/src/main/java/org/opensearch/common/collect/Iterators.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update libs/compress/src/main/java/org/opensearch/compress/ZstdCompressor.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Apply suggestions from code review Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update CollectionUtils.java Signed-off-by: Laurent Laborde * fixing javadoc warnings Signed-off-by: Laurent Laborde * remove useless p tag Signed-off-by: Laurent Laborde * removing a p tag Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * proper capitalization Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * improvements of existing javadocs Signed-off-by: Laurent Laborde * javadoc improvement Signed-off-by: Laurent Laborde * javadoc improvement Signed-off-by: Laurent Laborde * Fix formatting error after merge conflict resolution in a javadoc Signed-off-by: Laurent Laborde * "code" tag ignore carriage return & "p", switching to a "pre" tag Signed-off-by: Laurent Laborde * fixing an overzealous precommit check error. Signed-off-by: Laurent Laborde * Update distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update server/src/main/java/org/opensearch/cluster/RestoreInProgress.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * Update test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * further improvement of javadoc after review & suggestions Signed-off-by: Laurent Laborde * fixing unreferenced @link in javadoc (precommit error) Signed-off-by: Laurent Laborde * removing p tag from internal comment Signed-off-by: Laurent Laborde * reverting javadoc tag in licence header Signed-off-by: Laurent Laborde * Update server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java Co-authored-by: Andriy Redko Signed-off-by: Laurent Laborde * conflict resolution Signed-off-by: Laurent Laborde --------- Signed-off-by: Laurent Laborde Co-authored-by: Andriy Redko --- .../org/opensearch/gradle/reaper/Reaper.java | 5 +- .../org/opensearch/gradle/BwcVersions.java | 8 +- .../gradle/LoggingOutputStream.java | 2 +- .../precommit/TestingConventionRule.java | 2 +- .../gradle/tar/SymbolicLinkPreservingTar.java | 2 +- .../gradle/test/rest/RestResourcesPlugin.java | 2 +- .../gradle/vagrant/VagrantMachine.java | 4 +- .../gradle/vagrant/VagrantShellTask.java | 2 +- .../gradle/pluginzip/PublishTests.java | 12 +-- .../gradle/test/GradleThreadsFilter.java | 2 +- .../gradle/test/JUnit3MethodProvider.java | 2 +- .../benchmark/metrics/SampleRecorder.java | 2 +- .../org/opensearch/client/ClusterClient.java | 2 +- .../opensearch/client/RequestConverters.java | 6 +- .../org/opensearch/client/TimedRequest.java | 2 +- .../client/indices/CreateIndexRequest.java | 16 +-- .../client/indices/PutMappingRequest.java | 8 +- .../opensearch/client/tasks/TaskGroup.java | 1 - .../client/AbstractRequestTestCase.java | 2 +- .../client/AbstractResponseTestCase.java | 2 +- .../client/BulkProcessorRetryIT.java | 2 +- .../IndicesClientDocumentationIT.java | 6 +- .../IngestClientDocumentationIT.java | 6 +- .../SnapshotClientDocumentationIT.java | 6 +- .../StoredScriptsDocumentationIT.java | 6 +- .../TasksClientDocumentationIT.java | 6 +- .../indices/RandomCreateIndexGenerator.java | 2 +- .../org/opensearch/client/RestClient.java | 2 +- .../RestClientDocumentation.java | 6 +- .../documentation/SnifferDocumentation.java | 6 +- .../tools/launchers/SystemJvmOptions.java | 18 ++-- .../plugins/InstallPluginCommand.java | 5 +- .../plugins/ProgressInputStream.java | 2 +- .../org/opensearch/upgrade/UpgradeTask.java | 2 +- .../missingdoclet/MissingDoclet.java | 2 +- .../main/java/org/opensearch/cli/Command.java | 2 +- .../java/org/opensearch/cli/Terminal.java | 2 +- .../org/opensearch/bootstrap/JarHell.java | 4 +- .../java/org/opensearch/common/Explicit.java | 2 +- .../opensearch/common/LocalTimeOffset.java | 2 +- .../java/org/opensearch/common/SetOnce.java | 2 +- .../org/opensearch/common/collect/Tuple.java | 1 - .../org/opensearch/common/io/PathUtils.java | 4 +- .../common/network/InetAddresses.java | 4 +- .../org/opensearch/common/unit/TimeValue.java | 8 +- .../org/opensearch/common/util/BitMixer.java | 6 +- .../src/main/java/org/opensearch/Build.java | 2 +- .../java/org/opensearch/LegacyESVersion.java | 2 +- .../org/opensearch/OpenSearchException.java | 8 +- .../core/action/ActionListener.java | 6 +- .../org/opensearch/core/common/Strings.java | 2 +- .../common/bytes/CompositeBytesReference.java | 2 +- .../common/io/stream/BytesStreamInput.java | 2 +- .../io/stream/NamedWriteableRegistry.java | 2 +- .../core/common/io/stream/StreamInput.java | 4 +- .../core/common/io/stream/StreamOutput.java | 2 +- .../core/common/settings/SecureString.java | 4 +- .../core/common/unit/ByteSizeUnit.java | 2 +- .../opensearch/core/compress/Compressor.java | 2 +- .../core/compress/CompressorRegistry.java | 4 +- .../core/compress/spi/CompressorProvider.java | 2 +- .../java/org/opensearch/core/index/Index.java | 4 +- .../opensearch/core/index/shard/ShardId.java | 2 +- .../core/xcontent/AbstractObjectParser.java | 14 +-- .../core/xcontent/MapXContentParser.java | 2 +- .../core/xcontent/XContentBuilder.java | 2 +- .../xcontent/XContentBuilderExtension.java | 6 +- .../core/xcontent/XContentParser.java | 6 +- .../core/xcontent/XContentParserUtils.java | 4 +- .../core/xcontent/XContentSubParser.java | 2 +- .../org/opensearch/dissect/DissectParser.java | 39 ++++---- .../java/org/opensearch/geometry/Circle.java | 1 + .../opensearch/geometry/utils/BitUtil.java | 4 +- .../opensearch/geometry/utils/Geohash.java | 6 +- .../main/java/org/opensearch/grok/Grok.java | 2 +- .../org/opensearch/grok/MatcherWatchdog.java | 2 +- .../org/opensearch/nio/ChannelContext.java | 4 +- .../java/org/opensearch/nio/NioSelector.java | 4 +- .../opensearch/nio/SocketChannelContext.java | 2 +- .../opensearch/telemetry/tracing/Tracer.java | 2 +- .../telemetry/tracing/http/HttpTracer.java | 2 +- .../common/xcontent/XContentParserTests.java | 2 +- .../matrix/stats/RunningStats.java | 4 +- .../common/CommonAnalysisModulePlugin.java | 2 +- .../KeywordMarkerTokenFilterFactory.java | 4 +- .../analysis/common/SnowballAnalyzer.java | 2 +- .../bucket/composite/GeoTileValuesSource.java | 2 +- .../geogrid/cells/BoundedCellValues.java | 2 +- .../ingest/common/FailProcessorException.java | 2 +- .../ingest/common/ForEachProcessor.java | 4 +- .../expression/ExpressionScriptEngine.java | 2 +- .../script/mustache/MustacheScriptEngine.java | 2 +- .../mustache/MustacheScriptEngineTests.java | 2 +- .../opensearch/painless/spi/Allowlist.java | 2 +- .../painless/spi/AllowlistClass.java | 4 +- .../painless/spi/AllowlistLoader.java | 12 +-- .../painless/spi/AllowlistMethod.java | 2 +- .../org/opensearch/painless/Compiler.java | 4 +- .../lookup/PainlessLookupUtility.java | 6 +- .../phase/PainlessSemanticAnalysisPhase.java | 8 +- .../painless/symbol/SemanticScope.java | 2 +- .../mapper/SearchAsYouTypeFieldMapper.java | 2 +- .../join/query/HasChildQueryBuilder.java | 2 +- .../rankeval/TransportRankEvalAction.java | 4 +- .../DiscountedCumulativeGainTests.java | 98 +++++++++---------- .../AbstractAsyncBulkByScrollAction.java | 2 +- .../BulkByScrollParallelizationHelper.java | 12 +-- .../netty4/Netty4HttpRequestSizeLimitIT.java | 2 +- .../http/netty4/Netty4HttpRequest.java | 2 +- .../transport/CopyBytesSocketChannel.java | 2 +- .../analysis/phonetic/HaasePhonetik.java | 4 +- .../analysis/phonetic/KoelnerPhonetik.java | 6 +- .../index/analysis/phonetic/Nysiis.java | 2 +- .../discovery/gce/GceDiscoveryTests.java | 2 - .../BasePerFieldCorrelationVectorsFormat.java | 4 +- .../query/CorrelationQueryBuilderTests.java | 1 + .../customsuggester/CustomSuggestion.java | 3 +- .../ExampleAllowlistedClass.java | 2 +- .../ExampleAllowlistedInstance.java | 2 +- .../ExamplePainlessAnnotation.java | 2 +- .../ExampleStaticMethodClass.java | 2 +- .../AnnotatedTextFieldMapper.java | 4 +- .../azure/AzureBlobStoreRepositoryTests.java | 2 +- .../azure/AzureStorageService.java | 8 +- .../azure/AzureStorageServiceTests.java | 3 +- ...eCloudStorageBlobStoreRepositoryTests.java | 2 +- .../hdfs/HdfsSecurityContext.java | 2 +- .../hdfs/HdfsClientThreadLeakFilter.java | 2 +- .../s3/S3BlobStoreRepositoryTests.java | 2 +- .../repositories/s3/S3ClientSettings.java | 2 +- .../s3/S3RetryingInputStream.java | 2 +- .../index/store/SmbMMapDirectoryTests.java | 6 +- .../index/store/SmbNIOFSDirectoryTests.java | 6 +- .../opensearch/http/nio/NioHttpRequest.java | 2 +- .../upgrades/QueryBuilderBWCIT.java | 2 +- .../bootstrap/SpawnerNoBootstrapTests.java | 2 +- .../packaging/test/PackagingTestCase.java | 2 +- .../packaging/util/FileMatcher.java | 2 +- .../packaging/util/Installation.java | 2 +- .../org/opensearch/upgrades/IndexingIT.java | 13 +-- .../opensearch/action/admin/HotThreadsIT.java | 4 +- .../cluster/node/tasks/AbstractTasksIT.java | 2 +- .../node/tasks/ConcurrentSearchTasksIT.java | 4 +- .../repositories/RepositoryBlocksIT.java | 2 +- .../cluster/snapshots/SnapshotBlocksIT.java | 2 +- .../action/bulk/BulkProcessorRetryIT.java | 2 +- .../action/ingest/AsyncIngestProcessorIT.java | 2 +- .../action/search/TransportSearchIT.java | 4 +- .../discovery/ClusterDisruptionIT.java | 2 +- .../SegmentReplicationAllocationIT.java | 2 +- .../indices/state/CloseIndexIT.java | 2 +- .../recovery/FullRollingRestartIT.java | 10 +- .../aggregations/bucket/DateHistogramIT.java | 2 +- .../aggregations/pipeline/MaxBucketIT.java | 4 +- .../aggregations/pipeline/MovAvgIT.java | 2 +- .../search/pit/DeletePitMultiNodeIT.java | 34 +++---- .../search/searchafter/SearchAfterIT.java | 4 +- .../search/sort/GeoDistanceSortBuilderIT.java | 50 +++++----- .../SharedClusterSnapshotRestoreIT.java | 2 +- .../ConcurrentSeqNoVersioningIT.java | 2 +- .../apache/lucene/index/OneMergeHelper.java | 4 +- .../queryparser/classic/XQueryParser.java | 2 +- .../grouping/CollapsingTopDocsCollector.java | 2 +- .../BoundedBreakIteratorScanner.java | 6 +- .../apache/lucene/util/packed/XPacked64.java | 4 +- .../util/packed/XPacked64SingleBlock.java | 4 +- .../lucene/util/packed/XPackedInts.java | 4 +- .../org/opensearch/action/AliasesRequest.java | 2 +- .../opensearch/action/DocWriteRequest.java | 2 +- .../opensearch/action/DocWriteResponse.java | 2 +- .../action/TaskOperationFailure.java | 2 +- .../ClusterAllocationExplainRequest.java | 2 +- .../tasks/get/TransportGetTaskAction.java | 2 +- .../TransportCleanupRepositoryAction.java | 2 +- .../get/GetRepositoriesRequest.java | 2 +- .../create/CreateSnapshotRequest.java | 4 +- .../restore/RestoreSnapshotRequest.java | 2 +- .../alias/get/BaseAliasesRequestBuilder.java | 2 +- .../indices/create/CreateIndexRequest.java | 10 +- .../list/ListDanglingIndicesResponse.java | 2 +- .../mapping/get/GetFieldMappingsRequest.java | 2 +- .../mapping/get/GetFieldMappingsResponse.java | 2 +- .../mapping/put/PutMappingRequest.java | 2 +- .../SegmentReplicationStatsRequest.java | 2 +- .../indices/rollover/RolloverRequest.java | 2 +- .../indices/rollover/RolloverResponse.java | 2 +- .../opensearch/action/bulk/BackoffPolicy.java | 6 +- .../action/bulk/BulkItemResponse.java | 2 +- .../opensearch/action/bulk/BulkProcessor.java | 2 +- .../opensearch/action/bulk/BulkRequest.java | 10 +- .../action/delete/DeleteRequest.java | 2 +- .../action/delete/TransportDeleteAction.java | 2 +- .../opensearch/action/index/IndexRequest.java | 10 +- .../action/index/TransportIndexAction.java | 2 +- .../action/ingest/IngestActionForwarder.java | 2 +- .../action/search/CreatePitController.java | 20 ++-- .../action/search/SearchPhaseController.java | 4 +- .../action/search/SearchRequest.java | 4 +- .../action/search/SearchRequestBuilder.java | 2 +- .../action/search/SearchResponseSections.java | 2 +- .../search/TransportDeletePitAction.java | 2 +- .../action/support/TransportAction.java | 2 +- .../support/nodes/BaseNodesRequest.java | 4 +- .../replication/ReplicationOperation.java | 2 +- .../TransportReplicationAction.java | 4 +- .../replication/TransportWriteAction.java | 4 +- .../single/shard/SingleShardRequest.java | 2 +- .../support/tasks/BaseTasksRequest.java | 2 +- .../action/update/UpdateRequest.java | 2 +- .../opensearch/bootstrap/BootstrapChecks.java | 2 +- .../bootstrap/JNAKernel32Library.java | 24 ++--- .../org/opensearch/bootstrap/OpenSearch.java | 4 +- .../AckedClusterStateTaskListener.java | 2 +- .../cluster/ClusterStateTaskExecutor.java | 4 +- .../org/opensearch/cluster/DiffableUtils.java | 8 +- .../cluster/InternalClusterInfoService.java | 2 +- .../opensearch/cluster/RestoreInProgress.java | 2 +- .../coordination/ClusterStatePublisher.java | 4 +- .../coordination/JoinTaskExecutor.java | 4 +- .../cluster/coordination/Reconfigurator.java | 2 +- .../cluster/health/ClusterShardHealth.java | 4 +- .../cluster/metadata/IndexGraveyard.java | 4 +- .../cluster/metadata/IndexMetadata.java | 4 +- .../metadata/IndexTemplateMetadata.java | 4 +- .../opensearch/cluster/metadata/Metadata.java | 2 +- .../metadata/MetadataCreateIndexService.java | 6 +- .../metadata/MetadataIndexStateService.java | 4 +- .../MetadataIndexTemplateService.java | 2 +- .../cluster/node/DiscoveryNodes.java | 12 +-- .../routing/IndexShardRoutingTable.java | 2 +- .../cluster/routing/RecoverySource.java | 2 +- .../cluster/routing/RoutingNodes.java | 14 +-- .../routing/ShardMovementStrategy.java | 2 +- .../cluster/routing/UnassignedInfo.java | 2 +- .../allocation/AwarenessReplicaBalance.java | 2 +- .../routing/allocation/ConstraintTypes.java | 4 +- .../allocation/IndexMetadataUpdater.java | 4 +- .../allocator/BalancedShardsAllocator.java | 2 +- .../allocator/LocalShardsBalancer.java | 2 +- .../allocation/allocator/ShardsAllocator.java | 4 +- .../allocation/command/AllocationCommand.java | 2 +- .../allocation/decider/AllocationDecider.java | 2 +- .../decider/DiskThresholdDecider.java | 12 +-- .../decider/SameShardAllocationDecider.java | 2 +- .../decider/ThrottlingAllocationDecider.java | 2 +- .../service/ClusterApplierService.java | 2 +- .../service/ClusterManagerTaskThrottler.java | 8 +- .../org/opensearch/common/Randomness.java | 2 +- .../common/blobstore/BlobContainer.java | 6 +- .../common/blobstore/fs/FsBlobContainer.java | 4 +- .../breaker/ChildMemoryCircuitBreaker.java | 2 +- .../org/opensearch/common/cache/Cache.java | 2 +- .../common/collect/CopyOnWriteHashMap.java | 8 +- .../opensearch/common/geo/GeoShapeType.java | 10 +- .../org/opensearch/common/geo/GeoUtils.java | 4 +- .../opensearch/common/geo/GeometryFormat.java | 2 +- .../common/geo/builders/PolygonBuilder.java | 10 +- .../common/geo/parsers/GeoJsonParser.java | 2 +- .../common/geo/parsers/GeoWKTParser.java | 2 +- .../opensearch/common/hash/MurmurHash3.java | 2 +- .../opensearch/common/inject/Initializer.java | 5 +- .../common/io/stream/Streamables.java | 4 +- .../common/joda/JodaDateMathParser.java | 2 +- .../JsonThrowablePatternConverter.java | 2 +- .../common/lucene/ShardCoreKeyMap.java | 2 +- .../opensearch/common/rounding/Rounding.java | 2 +- .../settings/AbstractScopedSettings.java | 4 +- .../common/settings/KeyStoreWrapper.java | 8 +- .../common/settings/SecureSetting.java | 6 +- .../opensearch/common/settings/Settings.java | 2 +- .../common/time/DateFormatters.java | 4 +- .../common/time/DateMathParser.java | 6 +- .../org/opensearch/common/time/DateUtils.java | 2 +- .../common/time/DateUtilsRounding.java | 6 +- .../common/time/JavaDateFormatter.java | 2 +- .../common/time/JavaDateMathParser.java | 2 +- .../common/util/BinarySearcher.java | 10 +- .../common/util/CancellableThreads.java | 2 +- .../opensearch/common/util/CuckooFilter.java | 44 ++++----- .../util/SetBackedScalingCuckooFilter.java | 4 +- .../opensearch/common/util/TokenBucket.java | 2 +- .../opensearch/common/util/URIPattern.java | 4 +- .../util/concurrent/ListenableFuture.java | 2 +- ...ioritizedOpenSearchThreadPoolExecutor.java | 5 +- .../common/util/concurrent/ThreadContext.java | 8 +- .../xcontent/LoggingDeprecationHandler.java | 2 +- .../common/xcontent/XContentHelper.java | 2 +- .../xcontent/support/XContentMapValues.java | 13 ++- .../discovery/FileBasedSeedHostsProvider.java | 4 +- .../SettingsBasedSeedHostsProvider.java | 2 +- .../java/org/opensearch/env/Environment.java | 4 +- .../org/opensearch/env/NodeEnvironment.java | 4 +- .../extensions/ExtensionReader.java | 2 +- .../gateway/BaseGatewayShardAllocator.java | 2 +- .../gateway/DanglingIndicesState.java | 2 +- .../opensearch/gateway/GatewayMetaState.java | 2 +- .../gateway/PersistedClusterStateService.java | 8 +- .../gateway/PrimaryShardAllocator.java | 12 +-- .../gateway/ReplicaShardAllocator.java | 2 +- .../remote/RemoteClusterStateService.java | 9 +- .../java/org/opensearch/http/CorsHandler.java | 2 +- .../identity/noop/NoopIdentityPlugin.java | 2 +- .../opensearch/identity/noop/NoopSubject.java | 2 +- .../identity/tokens/RestTokenExtractor.java | 2 +- .../org/opensearch/index/IndexService.java | 4 +- .../org/opensearch/index/IndexSettings.java | 4 +- .../org/opensearch/index/IndexSortConfig.java | 2 +- .../index/MergeSchedulerConfig.java | 4 +- .../index/ShardIndexingPressure.java | 2 +- .../ShardIndexingPressureMemoryManager.java | 8 +- .../index/ShardIndexingPressureStore.java | 10 +- .../index/ShardIndexingPressureTracker.java | 6 +- .../org/opensearch/index/VersionType.java | 4 +- .../index/analysis/AnalysisRegistry.java | 2 +- .../NormalizingCharFilterFactory.java | 2 +- .../NormalizingTokenFilterFactory.java | 2 +- .../PreBuiltAnalyzerProviderFactory.java | 2 +- .../analysis/ShingleTokenFilterFactory.java | 10 +- .../index/analysis/TokenFilterFactory.java | 4 +- .../index/engine/InternalEngine.java | 2 +- .../index/engine/NRTReplicationEngine.java | 2 +- .../index/engine/ReplicaFileTracker.java | 2 +- .../index/fielddata/MultiGeoPointValues.java | 2 +- .../index/fielddata/ScriptDocValues.java | 12 +-- .../SortedSetOrdinalsIndexFieldData.java | 6 +- .../mapper/AbstractGeometryFieldMapper.java | 4 +- .../index/mapper/ArraySourceValueFetcher.java | 2 +- .../index/mapper/CompletionFieldMapper.java | 4 +- .../index/mapper/CustomDocValuesField.java | 3 +- .../index/mapper/DocumentMapperParser.java | 3 +- .../index/mapper/DynamicKeyFieldMapper.java | 2 +- .../mapper/DynamicKeyFieldTypeLookup.java | 4 +- .../index/mapper/FieldAliasMapper.java | 2 +- .../opensearch/index/mapper/FieldMapper.java | 2 +- .../index/mapper/FieldTypeLookup.java | 4 +- .../index/mapper/FlatObjectFieldMapper.java | 6 +- .../index/mapper/GeoPointFieldMapper.java | 2 +- .../index/mapper/MappedFieldType.java | 4 +- .../org/opensearch/index/mapper/Mapper.java | 2 +- .../index/mapper/MappingLookup.java | 2 +- .../index/mapper/MetadataFieldMapper.java | 2 +- .../index/mapper/ParametrizedFieldMapper.java | 8 +- .../index/mapper/SeqNoFieldMapper.java | 4 +- .../index/mapper/SourceValueFetcher.java | 2 +- .../opensearch/index/mapper/ValueFetcher.java | 4 +- .../index/query/AbstractQueryBuilder.java | 2 +- .../query/GeoBoundingBoxQueryBuilder.java | 2 +- .../index/query/GeoShapeQueryBuilder.java | 2 +- .../index/query/GeoValidationMethod.java | 2 +- .../index/query/IntervalsSourceProvider.java | 2 +- .../query/MatchBoolPrefixQueryBuilder.java | 2 +- .../index/query/MatchQueryBuilder.java | 2 +- .../index/query/MoreLikeThisQueryBuilder.java | 2 +- .../index/query/MultiMatchQueryBuilder.java | 4 +- .../index/query/QueryShardContext.java | 2 +- .../index/query/QueryStringQueryBuilder.java | 2 +- .../index/query/RangeQueryBuilder.java | 6 +- .../index/query/SearchIndexNameMatcher.java | 2 +- .../index/query/SimpleQueryStringBuilder.java | 2 +- .../index/reindex/BulkByScrollTask.java | 6 +- .../index/reindex/DeleteByQueryRequest.java | 2 +- .../opensearch/index/search/MatchQuery.java | 6 +- .../index/seqno/LocalCheckpointTracker.java | 4 +- .../index/seqno/ReplicationTracker.java | 10 +- .../index/shard/IndexEventListener.java | 2 +- .../opensearch/index/shard/IndexShard.java | 10 +- .../index/shard/RefreshListeners.java | 4 +- .../index/similarity/SimilarityProvider.java | 24 ++--- .../RemoteStoreShardShallowCopySnapshot.java | 8 +- .../org/opensearch/index/store/Store.java | 15 +-- .../remote/file/OnDemandBlockIndexInput.java | 6 +- .../DefaultTranslogDeletionPolicy.java | 2 +- .../opensearch/index/translog/Translog.java | 4 +- .../index/translog/TranslogWriter.java | 2 +- .../opensearch/indices/IndicesService.java | 6 +- .../indices/ShardLimitValidator.java | 2 +- .../analysis/PreBuiltCacheFactory.java | 10 +- .../recovery/RecoveryTargetHandler.java | 2 +- .../recovery/RemoteRecoveryTargetHandler.java | 2 +- .../OngoingSegmentReplications.java | 16 +-- .../common/ReplicationRequestTracker.java | 3 +- .../opensearch/ingest/ConfigurationUtils.java | 22 ++--- .../org/opensearch/ingest/IngestService.java | 2 +- .../java/org/opensearch/ingest/Pipeline.java | 4 +- .../java/org/opensearch/ingest/Processor.java | 4 +- .../org/opensearch/ingest/ValueSource.java | 2 +- .../org/opensearch/monitor/os/OsProbe.java | 10 +- .../NodePersistentTasksExecutor.java | 2 +- .../PersistentTasksClusterService.java | 2 +- .../plugins/CircuitBreakerPlugin.java | 6 +- .../org/opensearch/plugins/ClusterPlugin.java | 4 +- .../opensearch/plugins/DiscoveryPlugin.java | 9 +- .../org/opensearch/plugins/EnginePlugin.java | 2 +- .../opensearch/plugins/ExtensiblePlugin.java | 4 +- .../opensearch/plugins/IdentityPlugin.java | 8 +- .../org/opensearch/plugins/IngestPlugin.java | 2 +- .../java/org/opensearch/plugins/Plugin.java | 2 +- .../opensearch/plugins/PluginsService.java | 2 +- .../opensearch/plugins/RepositoryPlugin.java | 10 +- .../plugins/SearchPipelinePlugin.java | 6 +- .../org/opensearch/plugins/SearchPlugin.java | 4 +- .../org/opensearch/repositories/IndexId.java | 2 +- .../blobstore/BlobStoreRepository.java | 2 +- .../org/opensearch/rest/RestController.java | 2 +- .../java/org/opensearch/script/Script.java | 18 ++-- .../org/opensearch/script/ScriptCache.java | 2 +- .../org/opensearch/script/ScriptContext.java | 2 +- .../script/ScriptLanguagesInfo.java | 4 +- .../org/opensearch/script/ScriptMetadata.java | 6 +- .../opensearch/script/StoredScriptSource.java | 4 +- .../org/opensearch/search/MultiValueMode.java | 32 +++--- .../opensearch/search/SearchExtBuilder.java | 4 +- .../java/org/opensearch/search/SearchHit.java | 2 +- .../org/opensearch/search/SearchModule.java | 2 +- .../search/aggregations/AggregatorBase.java | 2 +- .../aggregations/InternalAggregations.java | 2 +- .../bucket/BucketsAggregator.java | 4 +- .../aggregations/bucket/GeoTileUtils.java | 2 +- .../MergingBucketsDeferringCollector.java | 15 +-- .../DateHistogramValuesSourceBuilder.java | 4 +- .../bucket/composite/InternalComposite.java | 12 +-- .../bucket/composite/ParsedComposite.java | 10 +- .../bucket/composite/SortedDocsProducer.java | 2 +- .../filter/FilterAggregatorFactory.java | 2 +- .../filter/FiltersAggregatorFactory.java | 2 +- .../AutoDateHistogramAggregator.java | 2 +- .../DateHistogramAggregationBuilder.java | 4 +- .../histogram/DateHistogramAggregator.java | 4 +- .../histogram/DateHistogramInterval.java | 2 +- .../histogram/DateIntervalConsumer.java | 2 +- .../bucket/histogram/DateIntervalWrapper.java | 8 +- .../bucket/histogram/DoubleBounds.java | 2 +- .../InternalVariableWidthHistogram.java | 4 +- .../bucket/histogram/LongBounds.java | 2 +- .../VariableWidthHistogramAggregator.java | 10 +- .../bucket/sampler/SamplerAggregator.java | 2 +- .../bucket/terms/InternalMultiTerms.java | 4 +- .../bucket/terms/InternalTerms.java | 10 +- .../bucket/terms/MultiTermsAggregator.java | 8 +- .../terms/RareTermsAggregationBuilder.java | 2 +- .../bucket/terms/TermsAggregator.java | 8 +- .../metrics/AbstractHyperLogLog.java | 6 +- .../metrics/AbstractHyperLogLogPlusPlus.java | 1 - .../metrics/AbstractLinearCounting.java | 8 +- ...AbstractPercentilesAggregationBuilder.java | 17 ++-- .../metrics/HyperLogLogPlusPlus.java | 8 +- .../metrics/HyperLogLogPlusPlusSparse.java | 2 +- .../metrics/ValueCountAggregator.java | 2 +- .../aggregations/pipeline/BucketHelpers.java | 2 +- .../pipeline/MovFnPipelineAggregator.java | 22 ++--- .../pipeline/MovingFunctions.java | 22 ++--- .../pipeline/SimulatedAnealingMinimizer.java | 6 +- .../support/AggregationInspectionHelper.java | 4 +- .../support/CoreValuesSourceType.java | 2 +- .../MultiValuesSourceAggregationBuilder.java | 2 +- .../ValuesSourceAggregationBuilder.java | 2 +- .../support/ValuesSourceType.java | 6 +- .../SearchBackpressureService.java | 4 +- .../settings/SearchBackpressureSettings.java | 6 +- .../search/fetch/FetchSubPhase.java | 4 +- .../fetch/subphase/FetchDocValuesPhase.java | 2 +- .../fetch/subphase/InnerHitsContext.java | 4 +- .../search/internal/ContextIndexSearcher.java | 2 +- .../search/internal/SearchContext.java | 2 +- .../search/lookup/SourceLookup.java | 2 +- .../pipeline/PipelineConfiguration.java | 2 +- .../opensearch/search/pipeline/Processor.java | 2 +- .../search/pipeline/ProcessorInfo.java | 2 +- .../profile/AbstractInternalProfileTree.java | 4 +- .../query/InternalProfileCollector.java | 13 +-- .../search/profile/query/QueryProfiler.java | 2 +- .../search/query/QuerySearchResult.java | 2 +- .../opensearch/search/rescore/Rescorer.java | 2 +- .../search/slice/DocValuesSliceQuery.java | 2 +- .../search/slice/TermsSliceQuery.java | 2 +- .../opensearch/search/suggest/Suggest.java | 2 +- .../completion/CompletionSuggestion.java | 4 +- .../suggest/completion/FuzzyOptions.java | 2 +- .../TopSuggestGroupDocsCollector.java | 4 +- .../completion/context/ContextMapping.java | 2 +- .../completion/context/ContextMappings.java | 10 +- .../completion/context/GeoContextMapping.java | 2 +- .../phrase/DirectCandidateGenerator.java | 14 +-- .../suggest/phrase/LinearInterpolation.java | 2 +- .../phrase/PhraseSuggestionBuilder.java | 2 +- .../snapshots/SnapshotsService.java | 8 +- .../main/java/org/opensearch/tasks/Task.java | 4 +- .../opensearch/tasks/TaskCancellation.java | 4 +- .../org/opensearch/threadpool/Scheduler.java | 4 +- .../threadpool/TaskAwareRunnable.java | 2 +- .../org/opensearch/threadpool/ThreadPool.java | 10 +- .../CompressibleBytesOutputStream.java | 4 +- .../transport/RemoteClusterConnection.java | 6 +- .../transport/TransportService.java | 8 +- .../org/opensearch/watcher/FileWatcher.java | 2 +- .../watcher/ResourceWatcherService.java | 2 +- .../opensearch/OpenSearchExceptionTests.java | 2 +- .../search/CreatePitControllerTests.java | 12 +-- .../TransportReplicationActionTests.java | 2 +- ...ReplicationAllPermitsAcquisitionTests.java | 2 +- .../allocation/AllocationPriorityTests.java | 2 +- .../allocation/BalanceConfigurationTests.java | 14 +-- .../allocation/CatAllocationTestCase.java | 2 +- ...dexShardConstraintDeciderOverlapTests.java | 2 +- .../allocation/IndexShardHotSpotTests.java | 2 +- .../RemoteShardsRebalanceShardsTests.java | 4 +- .../org/opensearch/common/RoundingTests.java | 2 +- .../common/geo/GeoJsonShapeParserTests.java | 6 +- .../common/geo/GeometryIndexerTests.java | 2 +- .../joda/JavaJodaTimeDuellingTests.java | 4 +- .../rounding/TimeZoneRoundingTests.java | 2 +- .../index/mapper/NestedObjectMapperTests.java | 4 +- .../index/query/InnerHitBuilderTests.java | 2 +- .../search/CreatePitSingleNodeTests.java | 14 +-- .../bucket/DateScriptMocksPlugin.java | 2 +- ...egacyIntervalCompositeAggBuilderTests.java | 2 +- .../metrics/MetricAggScriptPlugin.java | 2 +- .../pipeline/AvgBucketAggregatorTests.java | 6 +- .../pipeline/SearchPipelineInfoTests.java | 2 +- .../action/support/ActionTestUtils.java | 2 +- .../org/opensearch/cli/CommandTestCase.java | 2 +- ...archAllocationWithConstraintsTestCase.java | 44 ++++----- .../coordination/LinearizabilityChecker.java | 10 +- .../common/logging/JsonLogsIntegTestCase.java | 4 +- .../index/MockEngineFactoryPlugin.java | 2 +- .../index/RandomCreateIndexGenerator.java | 2 +- .../index/mapper/FieldMapperTestCase.java | 4 +- .../index/mapper/FieldMapperTestCase2.java | 4 +- .../index/shard/IndexShardTestCase.java | 2 +- .../mockito/plugin/PriviledgedMockMaker.java | 2 +- ...chMockAPIBasedRepositoryIntegTestCase.java | 6 +- .../opensearch/script/MockScriptEngine.java | 6 +- .../org/opensearch/script/ScoreAccessor.java | 2 +- .../aggregations/AggregatorTestCase.java | 10 +- ...AbstractDiffableSerializationTestCase.java | 4 +- ...ractDiffableWireSerializationTestCase.java | 4 +- .../test/AbstractQueryTestCase.java | 10 +- .../opensearch/test/AbstractWireTestCase.java | 4 +- .../opensearch/test/MockKeywordPlugin.java | 2 +- .../test/OpenSearchIntegTestCase.java | 12 +-- .../test/OpenSearchTokenStreamTestCase.java | 6 +- .../java/org/opensearch/test/TestCluster.java | 2 +- .../opensearch/test/XContentTestUtils.java | 12 +-- .../opensearch/test/client/NoOpClient.java | 2 +- .../test/client/NoOpNodeClient.java | 2 +- .../test/disruption/LongGCDisruption.java | 2 +- .../test/disruption/NetworkDisruption.java | 2 +- .../test/gateway/TestGatewayAllocator.java | 6 +- .../junit/annotations/TestIssueLogging.java | 2 +- .../test/junit/annotations/TestLogging.java | 2 +- .../test/junit/listeners/LoggingListener.java | 2 +- .../test/rest/RestActionTestCase.java | 2 +- .../yaml/DenylistedPathPatternMatcher.java | 2 +- .../opensearch/test/rest/yaml/Features.java | 2 +- .../OpenSearchClientYamlSuiteTestCase.java | 6 +- .../test/rest/yaml/section/DoSection.java | 2 +- .../section/GreaterThanEqualToAssertion.java | 2 +- .../rest/yaml/section/IsFalseAssertion.java | 2 +- .../rest/yaml/section/IsTrueAssertion.java | 2 +- .../rest/yaml/section/LessThanAssertion.java | 2 +- .../section/LessThanOrEqualToAssertion.java | 2 +- .../rest/yaml/section/MatchAssertion.java | 2 +- .../test/rest/yaml/section/SetSection.java | 2 +- 563 files changed, 1304 insertions(+), 1319 deletions(-) diff --git a/buildSrc/reaper/src/main/java/org/opensearch/gradle/reaper/Reaper.java b/buildSrc/reaper/src/main/java/org/opensearch/gradle/reaper/Reaper.java index c5b4de157c75c..662510fbbf61c 100644 --- a/buildSrc/reaper/src/main/java/org/opensearch/gradle/reaper/Reaper.java +++ b/buildSrc/reaper/src/main/java/org/opensearch/gradle/reaper/Reaper.java @@ -45,17 +45,16 @@ /** * A standalone process that will reap external services after a build dies. - * *

Input

* Since how to reap a given service is platform and service dependent, this tool * operates on system commands to execute. It takes a single argument, a directory * that will contain files with reaping commands. Each line in each file will be * executed with {@link Runtime#exec(String)}. - * + *

* The main method will wait indefinitely on the parent process (Gradle) by * reading from stdin. When Gradle shuts down, whether normally or abruptly, the * pipe will be broken and read will return. - * + *

* The reaper will then iterate over the files in the configured directory, * and execute the given commands. If any commands fail, a failure message is * written to stderr. Otherwise, the input file will be deleted. If no inputs diff --git a/buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java b/buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java index cddd03ccc2019..4d45640b75e3d 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java @@ -52,15 +52,15 @@ /** * A container for opensearch supported version information used in BWC testing. - * + *

* Parse the Java source file containing the versions declarations and use the known rules to figure out which are all * the version the current one is wire and index compatible with. * On top of this, figure out which of these are unreleased and provide the branch they can be built from. - * + *

* Note that in this context, currentVersion is the unreleased version this build operates on. * At any point in time there will surely be four such unreleased versions being worked on, * thus currentVersion will be one of these. - * + *

* Considering: *

*
M, M > 0
@@ -84,7 +84,7 @@ * Each build is only concerned with versions before it, as those are the ones that need to be tested * for backwards compatibility. We never look forward, and don't add forward facing version number to branches of previous * version. - * + *

* Each branch has a current version, and expected compatible versions are parsed from the server code's Version` class. * We can reliably figure out which the unreleased versions are due to the convention of always adding the next unreleased * version number to server in all branches when a version is released. diff --git a/buildSrc/src/main/java/org/opensearch/gradle/LoggingOutputStream.java b/buildSrc/src/main/java/org/opensearch/gradle/LoggingOutputStream.java index 5ae7ad1595e2f..5259700b3a63d 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/LoggingOutputStream.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/LoggingOutputStream.java @@ -38,7 +38,7 @@ /** * Writes data passed to this stream as log messages. - * + *

* The stream will be flushed whenever a newline is detected. * Allows setting an optional prefix before each line of output. */ diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionRule.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionRule.java index aa81ef75701fa..db46d2e3edc55 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionRule.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionRule.java @@ -40,7 +40,7 @@ /** * Represent rules for tests enforced by the @{link {@link TestingConventionsTasks}} - * + *

* Rules are identified by name, tests must have this name as a suffix and implement one of the base classes * and be part of all the specified tasks. */ diff --git a/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java b/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java index 1423b52c443d9..e82d8ed73ced2 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java @@ -61,7 +61,7 @@ /** * A custom archive task that assembles a tar archive that preserves symbolic links. - * + *

* This task is necessary because the built-in task {@link org.gradle.api.tasks.bundling.Tar} does not preserve symbolic links. */ public class SymbolicLinkPreservingTar extends Tar { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java index 728e36ce98bff..fcadf35593ce6 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java @@ -65,7 +65,7 @@ * Rest YAML tests :
* When the {@link RestResourcesPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly * configured with `includeCore` through the `restResources.restTests` extension. - * + *

* Additionally you can specify which sourceSetName resources should be copied to. The default is the yamlRestTest source set. * @see CopyRestApiTask * @see CopyRestTestsTask diff --git a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantMachine.java b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantMachine.java index 2d71b9361963b..7abf9bf5fbef6 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantMachine.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantMachine.java @@ -53,7 +53,7 @@ /** * An helper to manage a vagrant box. - * + *

* This is created alongside a {@link VagrantExtension} for a project to manage starting and * stopping a single vagrant box. */ @@ -185,7 +185,7 @@ public void setArgs(String... args) { /** * A function to translate output from the vagrant command execution to the progress line. - * + *

* The function takes the current line of output from vagrant, and returns a new * progress line, or {@code null} if there is no update. */ diff --git a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantShellTask.java b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantShellTask.java index 85d3e340c50e7..ca1b95183505f 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantShellTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/vagrant/VagrantShellTask.java @@ -47,7 +47,7 @@ /** * A shell script to run within a vagrant VM. - * + *

* The script is run as root within the VM. */ public abstract class VagrantShellTask extends DefaultTask { diff --git a/buildSrc/src/test/java/org/opensearch/gradle/pluginzip/PublishTests.java b/buildSrc/src/test/java/org/opensearch/gradle/pluginzip/PublishTests.java index c0e7320ba7615..8e246ff9ecd11 100644 --- a/buildSrc/src/test/java/org/opensearch/gradle/pluginzip/PublishTests.java +++ b/buildSrc/src/test/java/org/opensearch/gradle/pluginzip/PublishTests.java @@ -58,7 +58,7 @@ public void tearDown() { * This test is used to verify that adding the 'opensearch.pluginzip' to the project * adds some other transitive plugins and tasks under the hood. This is basically * a behavioral test of the {@link Publish#apply(Project)} method. - * + *

* This is equivalent of having a build.gradle script with just the following section: * 

      *     plugins {
@@ -202,7 +202,7 @@ public void useDefaultValues() throws IOException, URISyntaxException, XmlPullPa
         GradleRunner runner = prepareGradleRunnerFromTemplate("useDefaultValues.gradle", "build", ZIP_PUBLISH_TASK);
         BuildResult result = runner.build();
 
-        /** Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
+        /* Check if build and ZIP_PUBLISH_TASK tasks have run well */
         assertEquals(SUCCESS, result.task(":" + "build").getOutcome());
         assertEquals(SUCCESS, result.task(":" + ZIP_PUBLISH_TASK).getOutcome());
 
@@ -277,7 +277,7 @@ public void allProjectsGroup() throws IOException, URISyntaxException, XmlPullPa
         GradleRunner runner = prepareGradleRunnerFromTemplate("allProjectsGroup.gradle", "build", ZIP_PUBLISH_TASK);
         BuildResult result = runner.build();
 
-        /** Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
+        /* Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
         assertEquals(SUCCESS, result.task(":" + "build").getOutcome());
         assertEquals(SUCCESS, result.task(":" + ZIP_PUBLISH_TASK).getOutcome());
 
@@ -312,7 +312,7 @@ public void groupPriorityLevel() throws IOException, URISyntaxException, XmlPull
         GradleRunner runner = prepareGradleRunnerFromTemplate("groupPriorityLevel.gradle", "build", ZIP_PUBLISH_TASK);
         BuildResult result = runner.build();
 
-        /** Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
+        /* Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
         assertEquals(SUCCESS, result.task(":" + "build").getOutcome());
         assertEquals(SUCCESS, result.task(":" + ZIP_PUBLISH_TASK).getOutcome());
 
@@ -348,7 +348,7 @@ public void missingPOMEntity() throws IOException, URISyntaxException, XmlPullPa
         GradleRunner runner = prepareGradleRunnerFromTemplate("missingPOMEntity.gradle", "build", ZIP_PUBLISH_TASK);
         BuildResult result = runner.build();
 
-        /** Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
+        /* Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
         assertEquals(SUCCESS, result.task(":" + "build").getOutcome());
         assertEquals(SUCCESS, result.task(":" + ZIP_PUBLISH_TASK).getOutcome());
 
@@ -395,7 +395,7 @@ public void customizedGroupValue() throws IOException, URISyntaxException, XmlPu
         GradleRunner runner = prepareGradleRunnerFromTemplate("customizedGroupValue.gradle", "build", ZIP_PUBLISH_TASK);
         BuildResult result = runner.build();
 
-        /** Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
+        /* Check if build and {@value ZIP_PUBLISH_TASK} tasks have run well */
         assertEquals(SUCCESS, result.task(":" + "build").getOutcome());
         assertEquals(SUCCESS, result.task(":" + ZIP_PUBLISH_TASK).getOutcome());
 
diff --git a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java
index b64c719440733..def5248c1f255 100644
--- a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java
+++ b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java
@@ -36,7 +36,7 @@
 
 /**
  * Filter out threads controlled by gradle that may be created during unit tests.
- *
+ * 

  * Currently this includes pooled threads for Exec as well as file system event watcher threads.
  */
 public class GradleThreadsFilter implements ThreadFilter {
diff --git a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/JUnit3MethodProvider.java b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/JUnit3MethodProvider.java
index 163a903d31832..1a2e36aa78e9f 100644
--- a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/JUnit3MethodProvider.java
+++ b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/JUnit3MethodProvider.java
@@ -43,7 +43,7 @@
 
 /**
  * Backwards compatible test* method provider (public, non-static).
- *
+ * 

  * copy of org.apache.lucene.util.LuceneJUnit3MethodProvider to avoid a dependency between build and test fw.
  */
 public final class JUnit3MethodProvider implements TestMethodProvider {
diff --git a/client/benchmark/src/main/java/org/opensearch/client/benchmark/metrics/SampleRecorder.java b/client/benchmark/src/main/java/org/opensearch/client/benchmark/metrics/SampleRecorder.java
index e53e4f1ad692d..9cd12f5e78bd0 100644
--- a/client/benchmark/src/main/java/org/opensearch/client/benchmark/metrics/SampleRecorder.java
+++ b/client/benchmark/src/main/java/org/opensearch/client/benchmark/metrics/SampleRecorder.java
@@ -37,7 +37,7 @@
 
 /**
  * Stores measurement samples.
- *
+ * 

  * This class is NOT threadsafe.
  */
 public final class SampleRecorder {
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java
index 5bd5a5d0e308e..eb0a8b0e8f40a 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java
@@ -170,8 +170,8 @@ public ClusterHealthResponse health(ClusterHealthRequest healthRequest, RequestO
 
     /**
      * Asynchronously get cluster health using the Cluster Health API.
-     *
      * If timeout occurred, {@link ClusterHealthResponse} will have isTimedOut() == true and status() == RestStatus.REQUEST_TIMEOUT
+     *
      * @param healthRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
index 61a202d25167f..35d9929a649ff 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
@@ -451,9 +451,9 @@ static void addSearchRequestParams(Params params, SearchRequest searchRequest) {
             params.withIndicesOptions(searchRequest.indicesOptions());
         }
         params.withSearchType(searchRequest.searchType().name().toLowerCase(Locale.ROOT));
-        /**
-         * Merging search responses as part of CCS flow to reduce roundtrips is not supported for point in time -
-         * refer to org.opensearch.action.search.SearchResponseMerger
+        /*
+          Merging search responses as part of CCS flow to reduce roundtrips is not supported for point in time -
+          refer to org.opensearch.action.search.SearchResponseMerger
          */
         if (searchRequest.pointInTimeBuilder() != null) {
             params.putParam("ccs_minimize_roundtrips", "false");
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/TimedRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/TimedRequest.java
index dad5b6a3679ec..d40445b2daa81 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/TimedRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/TimedRequest.java
@@ -37,7 +37,7 @@
 
 /**
  * A base request for any requests that supply timeouts.
- *
+ * 

  * Please note, any requests that use a ackTimeout should set timeout as they
  * represent the same backing field on the server.
  */
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java
index 7805a7853b003..62c5b54c0e75e 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java
@@ -156,7 +156,7 @@ public MediaType mappingsMediaType() {
 
     /**
      * Adds mapping that will be added when the index gets created.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      *
      * @param source The mapping source
@@ -168,7 +168,7 @@ public CreateIndexRequest mapping(String source, MediaType mediaType) {
 
     /**
      * Adds mapping that will be added when the index gets created.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      *
      * @param source The mapping source
@@ -179,7 +179,7 @@ public CreateIndexRequest mapping(XContentBuilder source) {
 
     /**
      * Adds mapping that will be added when the index gets created.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      *
      * @param source The mapping source
@@ -196,7 +196,7 @@ public CreateIndexRequest mapping(Map source) {
 
     /**
      * Adds mapping that will be added when the index gets created.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      *
      * @param source The mapping source
@@ -282,7 +282,7 @@ public CreateIndexRequest aliases(Collection aliases) {
 
     /**
      * Sets the settings and mappings as a single source.
-     *
+     * 

      * Note that the mapping definition should *not* be nested under a type name.
      */
     public CreateIndexRequest source(String source, MediaType mediaType) {
@@ -291,7 +291,7 @@ public CreateIndexRequest source(String source, MediaType mediaType) {
 
     /**
      * Sets the settings and mappings as a single source.
-     *
+     * 

      * Note that the mapping definition should *not* be nested under a type name.
      */
     public CreateIndexRequest source(XContentBuilder source) {
@@ -300,7 +300,7 @@ public CreateIndexRequest source(XContentBuilder source) {
 
     /**
      * Sets the settings and mappings as a single source.
-     *
+     * 

      * Note that the mapping definition should *not* be nested under a type name.
      */
     public CreateIndexRequest source(BytesReference source, MediaType mediaType) {
@@ -311,7 +311,7 @@ public CreateIndexRequest source(BytesReference source, MediaType mediaType) {
 
     /**
      * Sets the settings and mappings as a single source.
-     *
+     * 

      * Note that the mapping definition should *not* be nested under a type name.
      */
     @SuppressWarnings("unchecked")
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java
index 6d7e95d191ba6..a63393bd2341b 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java
@@ -105,7 +105,7 @@ public MediaType mediaType() {
 
     /**
      * The mapping source definition.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      */
     public PutMappingRequest source(Map mappingSource) {
@@ -120,7 +120,7 @@ public PutMappingRequest source(Map mappingSource) {
 
     /**
      * The mapping source definition.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      */
     public PutMappingRequest source(String mappingSource, MediaType mediaType) {
@@ -131,7 +131,7 @@ public PutMappingRequest source(String mappingSource, MediaType mediaType) {
 
     /**
      * The mapping source definition.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      */
     public PutMappingRequest source(XContentBuilder builder) {
@@ -142,7 +142,7 @@ public PutMappingRequest source(XContentBuilder builder) {
 
     /**
      * The mapping source definition.
-     *
+     * 

      * Note that the definition should *not* be nested under a type name.
      */
     public PutMappingRequest source(BytesReference source, MediaType mediaType) {
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskGroup.java b/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskGroup.java
index c419884700587..9129de717459f 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskGroup.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskGroup.java
@@ -38,7 +38,6 @@
 
 /**
  * Client side counterpart of server side version.
- *
  * {@link org.opensearch.action.admin.cluster.node.tasks.list.TaskGroup}
  */
 public class TaskGroup {
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java
index 49bcb61b2dc3d..c464ee9ece74a 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java
@@ -44,7 +44,7 @@
 
 /**
  * Base class for HLRC request parsing tests.
- *
+ * 

  * This case class facilitates generating client side request test instances and
  * verifies that they are correctly parsed into server side request instances.
  *
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java
index 27704b01560c4..7d2d6b87b85c6 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java
@@ -44,7 +44,7 @@
 
 /**
  * Base class for HLRC response parsing tests.
- *
+ * 

  * This case class facilitates generating server side response test instances and
  * verifies that they are correctly parsed into HLRC response instances.
  *
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java
index b7f6328b3c88e..3678cc042ba47 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorRetryIT.java
@@ -180,7 +180,7 @@ private static MultiGetRequest indexDocs(BulkProcessor processor, int numDocs) {
     /**
      * Internal helper class to correlate backoff states with bulk responses. This is needed to check whether we maxed out the number
      * of retries but still got rejected (which is perfectly fine and can also happen from time to time under heavy load).
-     *
+     * 

      * This implementation relies on an implementation detail in Retry, namely that the bulk listener is notified on the same thread
      * as the last call to the backoff policy's iterator. The advantage is that this is non-invasive to the rest of the production code.
      */
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java
index abb2d75aea751..ce080b45273b4 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java
@@ -137,15 +137,15 @@
  * You need to wrap your code between two tags like:
  * // tag::example
  * // end::example
- *
+ * 

  * Where example is your tag name.
- *
+ * 

  * Then in the documentation, you can extract what is between tag and end tags with
  * ["source","java",subs="attributes,callouts,macros"]
  * --------------------------------------------------
  * include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[example]
  * --------------------------------------------------
- *
+ * 

  * The column width of the code block is 84. If the code contains a line longer
  * than 84, the line will be cut and a horizontal scroll bar will be displayed.
  * (the code indentation of the tag is not included in the width)
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java
index d14759065b5eb..28909cf58541a 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java
@@ -65,15 +65,15 @@
  * You need to wrap your code between two tags like:
  * // tag::example
  * // end::example
- *
+ * 

  * Where example is your tag name.
- *
+ * 

  * Then in the documentation, you can extract what is between tag and end tags with
  * ["source","java",subs="attributes,callouts,macros"]
  * --------------------------------------------------
  * include-tagged::{doc-tests}/IngestClientDocumentationIT.java[example]
  * --------------------------------------------------
- *
+ * 

  * The column width of the code block is 84. If the code contains a line longer
  * than 84, the line will be cut and a horizontal scroll bar will be displayed.
  * (the code indentation of the tag is not included in the width)
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java
index 50bcf79642eac..d0015db044843 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java
@@ -90,15 +90,15 @@
  * You need to wrap your code between two tags like:
  * // tag::example
  * // end::example
- *
+ * 

  * Where example is your tag name.
- *
+ * 

  * Then in the documentation, you can extract what is between tag and end tags with
  * ["source","java",subs="attributes,callouts,macros"]
  * --------------------------------------------------
  * include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[example]
  * --------------------------------------------------
- *
+ * 

  * The column width of the code block is 84. If the code contains a line longer
  * than 84, the line will be cut and a horizontal scroll bar will be displayed.
  * (the code indentation of the tag is not included in the width)
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java
index 6916ae11556e2..2e2d15df5392a 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java
@@ -66,15 +66,15 @@
  * You need to wrap your code between two tags like:
  * // tag::example
  * // end::example
- *
+ * 

  * Where example is your tag name.
- *
+ * 

  * Then in the documentation, you can extract what is between tag and end tags with
  * ["source","java",subs="attributes,callouts,macros"]
  * --------------------------------------------------
  * include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[example]
  * --------------------------------------------------
- *
+ * 

  * The column width of the code block is 84. If the code contains a line longer
  * than 84, the line will be cut and a horizontal scroll bar will be displayed.
  * (the code indentation of the tag is not included in the width)
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/TasksClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/TasksClientDocumentationIT.java
index 03e267aafd1b7..cbac0b8c97d9c 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/TasksClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/TasksClientDocumentationIT.java
@@ -66,15 +66,15 @@
  * You need to wrap your code between two tags like:
  * // tag::example
  * // end::example
- *
+ * 

  * Where example is your tag name.
- *
+ * 

  * Then in the documentation, you can extract what is between tag and end tags with
  * ["source","java",subs="attributes,callouts,macros"]
  * --------------------------------------------------
  * include-tagged::{doc-tests}/{@link TasksClientDocumentationIT}.java[example]
  * --------------------------------------------------
- *
+ * 

  * The column width of the code block is 84. If the code contains a line longer
  * than 84, the line will be cut and a horizontal scroll bar will be displayed.
  * (the code indentation of the tag is not included in the width)
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/RandomCreateIndexGenerator.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/RandomCreateIndexGenerator.java
index 1f747dc139d15..edb4d16c6d992 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/RandomCreateIndexGenerator.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/RandomCreateIndexGenerator.java
@@ -44,7 +44,7 @@ public class RandomCreateIndexGenerator {
 
     /**
      * Returns a random {@link CreateIndexRequest}.
-     *
+     * 

      * Randomizes the index name, the aliases, mappings and settings associated with the
      * index. When present, the mappings make no mention of types.
      */
diff --git a/client/rest/src/main/java/org/opensearch/client/RestClient.java b/client/rest/src/main/java/org/opensearch/client/RestClient.java
index e819fa27a8939..7691c01daefea 100644
--- a/client/rest/src/main/java/org/opensearch/client/RestClient.java
+++ b/client/rest/src/main/java/org/opensearch/client/RestClient.java
@@ -310,7 +310,7 @@ public boolean isRunning() {
      * they will be retried). In case of failures all of the alive nodes (or
      * dead nodes that deserve a retry) are retried until one responds or none
      * of them does, in which case an {@link IOException} will be thrown.
-     *
+     * 

      * This method works by performing an asynchronous call and waiting
      * for the result. If the asynchronous call throws an exception we wrap
      * it and rethrow it so that the stack trace attached to the exception
diff --git a/client/rest/src/test/java/org/opensearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/opensearch/client/documentation/RestClientDocumentation.java
index b2807d35d230e..42c31864e0578 100644
--- a/client/rest/src/test/java/org/opensearch/client/documentation/RestClientDocumentation.java
+++ b/client/rest/src/test/java/org/opensearch/client/documentation/RestClientDocumentation.java
@@ -89,15 +89,15 @@
  * You need to wrap your code between two tags like:
  * // tag::example[]
  * // end::example[]
- *
+ * 

  * Where example is your tag name.
- *
+ * 

  * Then in the documentation, you can extract what is between tag and end tags with
  * ["source","java",subs="attributes,callouts,macros"]
  * --------------------------------------------------
  * include-tagged::{doc-tests}/RestClientDocumentation.java[example]
  * --------------------------------------------------
- *
+ * 

  * Note that this is not a test class as we are only interested in testing that docs snippets compile. We don't want
  * to send requests to a node and we don't even have the tools to do it.
  */
diff --git a/client/sniffer/src/test/java/org/opensearch/client/sniff/documentation/SnifferDocumentation.java b/client/sniffer/src/test/java/org/opensearch/client/sniff/documentation/SnifferDocumentation.java
index 440e9a2ea5cd1..8a4ca1fb0a136 100644
--- a/client/sniffer/src/test/java/org/opensearch/client/sniff/documentation/SnifferDocumentation.java
+++ b/client/sniffer/src/test/java/org/opensearch/client/sniff/documentation/SnifferDocumentation.java
@@ -49,15 +49,15 @@
  * You need to wrap your code between two tags like:
  * // tag::example[]
  * // end::example[]
- *
+ * 

  * Where example is your tag name.
- *
+ * 

  * Then in the documentation, you can extract what is between tag and end tags with
  * ["source","java",subs="attributes,callouts,macros"]
  * --------------------------------------------------
  * include-tagged::{doc-tests}/SnifferDocumentation.java[example]
  * --------------------------------------------------
- *
+ * 

  * Note that this is not a test class as we are only interested in testing that docs snippets compile. We don't want
  * to send requests to a node and we don't even have the tools to do it.
  */
diff --git a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java
index aa3dfbe39ee96..726c381db09f6 100644
--- a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java
+++ b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java
@@ -101,15 +101,15 @@ private static String maybeShowCodeDetailsInExceptionMessages() {
     }
 
     private static String javaLocaleProviders() {
-        /**
-         *  SPI setting is used to allow loading custom CalendarDataProvider
-         *  in jdk8 it has to be loaded from jre/lib/ext,
-         *  in jdk9+ it is already within ES project and on a classpath
-         *
-         *  Due to internationalization enhancements in JDK 9 OpenSearch need to set the provider to COMPAT otherwise time/date
-         *  parsing will break in an incompatible way for some date patterns and locales.
-         *  //TODO COMPAT will be deprecated in at some point, see please https://bugs.openjdk.java.net/browse/JDK-8232906
-         * See also: documentation in server/org.opensearch.common.time.IsoCalendarDataProvider
+        /*
+           SPI setting is used to allow loading custom CalendarDataProvider
+           in jdk8 it has to be loaded from jre/lib/ext,
+           in jdk9+ it is already within ES project and on a classpath
+
+           Due to internationalization enhancements in JDK 9 OpenSearch need to set the provider to COMPAT otherwise time/date
+           parsing will break in an incompatible way for some date patterns and locales.
+           //TODO COMPAT will be deprecated in at some point, see please https://bugs.openjdk.java.net/browse/JDK-8232906
+          See also: documentation in server/org.opensearch.common.time.IsoCalendarDataProvider
          */
         return "-Djava.locale.providers=SPI,COMPAT";
     }
diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java
index 66f43b1e30d28..838d6e22a37bd 100644
--- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java
+++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java
@@ -107,7 +107,7 @@
 
 /**
  * A command for the plugin cli to install a plugin into opensearch.
- *
+ * 

  * The install command takes a plugin id, which may be any of the following:
  * 
    
  * 
  • An official opensearch plugin name
    • 
@@ -411,7 +411,7 @@ private String getMavenUrl(Terminal terminal, String[] coordinates, String platf
 
     /**
      * Returns {@code true} if the given url exists, and {@code false} otherwise.
-     *
+     * 
    
      * The given url must be {@code https} and existing means a {@code HEAD} request returns 200.
      */
     // pkg private for tests to manipulate
@@ -698,7 +698,6 @@ InputStream getPublicKey() {
 
     /**
      * Creates a URL and opens a connection.
-     *
      * If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned.
      */
     // pkg private for tests
diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ProgressInputStream.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ProgressInputStream.java
index 579f676631a5a..02be3dbc82a44 100644
--- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ProgressInputStream.java
+++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ProgressInputStream.java
@@ -41,7 +41,7 @@
  * The listener is triggered whenever a full percent is increased
  * The listener is never triggered twice on the same percentage
  * The listener will always return 99 percent, if the expectedTotalSize is exceeded, until it is finished
- *
+ * 
    
  * Only used by the InstallPluginCommand, thus package private here
  */
 abstract class ProgressInputStream extends FilterInputStream {
diff --git a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/UpgradeTask.java b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/UpgradeTask.java
index b7dcbd50cf781..708f644bcdeb6 100644
--- a/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/UpgradeTask.java
+++ b/distribution/tools/upgrade-cli/src/main/java/org/opensearch/upgrade/UpgradeTask.java
@@ -17,7 +17,7 @@
  * An interface for an upgrade task, which in this instance is an unit of
  * operation that is part of the overall upgrade process. This extends the
  * {@link java.util.function.Consumer} interface.
- *
+ * 
    
  * The implementing tasks consume and instance of a tuple of {@link TaskInput}
  * and {@link Terminal} and operate via side effects.
  *
diff --git a/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java b/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java
index e6122e7baf91a..e1ad55fe4b60b 100644
--- a/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java
+++ b/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java
@@ -45,7 +45,7 @@
  *   It isn't recursive, just ignores exactly the elements you tell it.
  * Has option --missing-method to apply "method" level to selected packages (fix one at a time).
  *   Matches package names exactly: so you'll need to list subpackages separately.
- *
+ * 
    
  * Note: This by default ignores javadoc validation on overridden methods.
  */
 // Original version of this class is ported from MissingDoclet code in Lucene,
diff --git a/libs/cli/src/main/java/org/opensearch/cli/Command.java b/libs/cli/src/main/java/org/opensearch/cli/Command.java
index eed5c4ba4ee6f..cc9230bdb2282 100644
--- a/libs/cli/src/main/java/org/opensearch/cli/Command.java
+++ b/libs/cli/src/main/java/org/opensearch/cli/Command.java
@@ -162,7 +162,7 @@ protected static void exit(int status) {
 
     /**
      * Executes this command.
-     *
+     * 
    
      * Any runtime user errors (like an input file that does not exist), should throw a {@link UserException}. */
     protected abstract void execute(Terminal terminal, OptionSet options) throws Exception;
 
diff --git a/libs/cli/src/main/java/org/opensearch/cli/Terminal.java b/libs/cli/src/main/java/org/opensearch/cli/Terminal.java
index be030c18507ad..fb1097178e5a3 100644
--- a/libs/cli/src/main/java/org/opensearch/cli/Terminal.java
+++ b/libs/cli/src/main/java/org/opensearch/cli/Terminal.java
@@ -44,7 +44,7 @@
 
 /**
  * A Terminal wraps access to reading input and writing output for a cli.
- *
+ * 
    
  * The available methods are similar to those of {@link Console}, with the ability
  * to read either normal text or a password, and the ability to print a line
  * of text. Printing is also gated by the {@link Verbosity} of the terminal,
diff --git a/libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java b/libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java
index c4ba778e7db86..fc5e364241d12 100644
--- a/libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java
+++ b/libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java
@@ -104,7 +104,7 @@ public static void checkJarHell(Consumer output) throws IOException, URI
 
     /**
      * Parses the classpath into an array of URLs
-     * @return array of URLs
+     * @return collection of URLs
      * @throws IllegalStateException if the classpath contains empty elements
      */
     public static Set parseClassPath() {
@@ -114,7 +114,7 @@ public static Set parseClassPath() {
     /**
      * Parses the classpath into a set of URLs. For testing.
      * @param classPath classpath to parse (typically the system property {@code java.class.path})
-     * @return array of URLs
+     * @return collection of URLs
      * @throws IllegalStateException if the classpath contains empty elements
      */
     @SuppressForbidden(reason = "resolves against CWD because that is how classpaths work")
diff --git a/libs/common/src/main/java/org/opensearch/common/Explicit.java b/libs/common/src/main/java/org/opensearch/common/Explicit.java
index 66e079c461e75..e060baf6f187e 100644
--- a/libs/common/src/main/java/org/opensearch/common/Explicit.java
+++ b/libs/common/src/main/java/org/opensearch/common/Explicit.java
@@ -38,7 +38,7 @@
  * Holds a value that is either:
  * a) set implicitly e.g. through some default value
  * b) set explicitly e.g. from a user selection
- *
+ * 
    
  * When merging conflicting configuration settings such as
  * field mapping settings it is preferable to preserve an explicit
  * choice rather than a choice made only made implicitly by defaults.
diff --git a/libs/common/src/main/java/org/opensearch/common/LocalTimeOffset.java b/libs/common/src/main/java/org/opensearch/common/LocalTimeOffset.java
index 7e89641927ed5..eb7b331c9aa24 100644
--- a/libs/common/src/main/java/org/opensearch/common/LocalTimeOffset.java
+++ b/libs/common/src/main/java/org/opensearch/common/LocalTimeOffset.java
@@ -514,7 +514,7 @@ public boolean anyMoveBackToPreviousDay() {
      * Builds an array that can be {@link Arrays#binarySearch(long[], long)}ed
      * for the daylight savings time transitions.
      *
-     * @openearch.internal
+     * @opensearch.internal
      */
     private static class TransitionArrayLookup extends AbstractManyTransitionsLookup {
         private final LocalTimeOffset[] offsets;
diff --git a/libs/common/src/main/java/org/opensearch/common/SetOnce.java b/libs/common/src/main/java/org/opensearch/common/SetOnce.java
index a596b5fcdb61d..778926ce108b7 100644
--- a/libs/common/src/main/java/org/opensearch/common/SetOnce.java
+++ b/libs/common/src/main/java/org/opensearch/common/SetOnce.java
@@ -35,7 +35,7 @@
  * A convenient class which offers a semi-immutable object wrapper implementation which allows one
  * to set the value of an object exactly once, and retrieve it many times. If {@link #set(Object)}
  * is called more than once, {@link AlreadySetException} is thrown and the operation will fail.
- *
+ * 
    
  * This is borrowed from lucene's experimental API. It is not reused to eliminate the dependency
  * on lucene core for such a simple (standalone) utility class that may change beyond OpenSearch needs.
  *
diff --git a/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java b/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java
index d0b94536b0729..5c0e3f2de7708 100644
--- a/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java
+++ b/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java
@@ -34,7 +34,6 @@
 
 /**
  * Java 9 Tuple
- *
  * todo: deprecate and remove w/ min jdk upgrade to 11?
  *
  * @opensearch.internal
diff --git a/libs/common/src/main/java/org/opensearch/common/io/PathUtils.java b/libs/common/src/main/java/org/opensearch/common/io/PathUtils.java
index b3526859933ec..ed8d50892b74a 100644
--- a/libs/common/src/main/java/org/opensearch/common/io/PathUtils.java
+++ b/libs/common/src/main/java/org/opensearch/common/io/PathUtils.java
@@ -93,7 +93,7 @@ public static Path get(URI uri) {
 
     /**
      * Tries to resolve the given path against the list of available roots.
-     *
+     * 
    
      * If path starts with one of the listed roots, it returned back by this method, otherwise null is returned.
      */
     public static Path get(Path[] roots, String path) {
@@ -109,7 +109,7 @@ public static Path get(Path[] roots, String path) {
 
     /**
      * Tries to resolve the given file uri against the list of available roots.
-     *
+     * 
    
      * If uri starts with one of the listed roots, it returned back by this method, otherwise null is returned.
      */
     public static Path get(Path[] roots, URI uri) {
diff --git a/libs/common/src/main/java/org/opensearch/common/network/InetAddresses.java b/libs/common/src/main/java/org/opensearch/common/network/InetAddresses.java
index a4fbc6cb65b0d..0f289c09bbae2 100644
--- a/libs/common/src/main/java/org/opensearch/common/network/InetAddresses.java
+++ b/libs/common/src/main/java/org/opensearch/common/network/InetAddresses.java
@@ -368,7 +368,7 @@ public static InetAddress forString(String ipString) {
 
     /**
      * Convert a byte array into an InetAddress.
-     *
+     * 
    
      * {@link InetAddress#getByAddress} is documented as throwing a checked
      * exception "if IP address is of illegal length."  We replace it with
      * an unchecked exception, for use by callers who already know that addr
@@ -423,7 +423,7 @@ public static Tuple parseCidr(String maskedAddress) {
 
     /**
      * Given an address and prefix length, returns the string representation of the range in CIDR notation.
-     *
+     * 
    
      * See {@link #toAddrString} for details on how the address is represented.
      */
     public static String toCidrString(InetAddress address, int prefixLength) {
diff --git a/libs/common/src/main/java/org/opensearch/common/unit/TimeValue.java b/libs/common/src/main/java/org/opensearch/common/unit/TimeValue.java
index a3fcffb1d6a4c..30ed5bf63a748 100644
--- a/libs/common/src/main/java/org/opensearch/common/unit/TimeValue.java
+++ b/libs/common/src/main/java/org/opensearch/common/unit/TimeValue.java
@@ -224,10 +224,10 @@ public double getDaysFrac() {
 
     /**
      * Returns a {@link String} representation of the current {@link TimeValue}.
-     *
+     * 
    
      * Note that this method might produce fractional time values (ex 1.6m) which cannot be
      * parsed by method like {@link TimeValue#parse(String, String, String, String)}.
-     *
+     * 
    
      * Also note that the maximum string value that will be generated is
      * {@code 106751.9d} due to the way that values are internally converted
      * to nanoseconds (106751.9 days is Long.MAX_VALUE nanoseconds)
@@ -239,12 +239,12 @@ public String toString() {
 
     /**
      * Returns a {@link String} representation of the current {@link TimeValue}.
-     *
+     * 
    
      * Note that this method might produce fractional time values (ex 1.6m) which cannot be
      * parsed by method like {@link TimeValue#parse(String, String, String, String)}. The number of
      * fractional decimals (up to 10 maximum) are truncated to the number of fraction pieces
      * specified.
-     *
+     * 
    
      * Also note that the maximum string value that will be generated is
      * {@code 106751.9d} due to the way that values are internally converted
      * to nanoseconds (106751.9 days is Long.MAX_VALUE nanoseconds)
diff --git a/libs/common/src/main/java/org/opensearch/common/util/BitMixer.java b/libs/common/src/main/java/org/opensearch/common/util/BitMixer.java
index 8762217916c7a..d6ea4fa359df3 100644
--- a/libs/common/src/main/java/org/opensearch/common/util/BitMixer.java
+++ b/libs/common/src/main/java/org/opensearch/common/util/BitMixer.java
@@ -25,9 +25,9 @@
 
 /**
  * Bit mixing utilities from carrotsearch.hppc.
- *
+ * 
    
  * Licensed under ALv2. This is pulled in directly to avoid a full hppc dependency.
- *
+ * 
    
  * The purpose of these methods is to evenly distribute key space over int32
  * range.
  */
@@ -111,7 +111,7 @@ public static int mix32(int k) {
     /**
      * Computes David Stafford variant 9 of 64bit mix function (MH3 finalization step,
      * with different shifts and constants).
-     *
+     * 
    
      * Variant 9 is picked because it contains two 32-bit shifts which could be possibly
      * optimized into better machine code.
      *
diff --git a/libs/core/src/main/java/org/opensearch/Build.java b/libs/core/src/main/java/org/opensearch/Build.java
index 67a50a8a31a0e..b5d67f5501725 100644
--- a/libs/core/src/main/java/org/opensearch/Build.java
+++ b/libs/core/src/main/java/org/opensearch/Build.java
@@ -216,7 +216,7 @@ public String getDistribution() {
 
     /**
      * Get the version as considered at build time
-     *
+     * 
    
      * Offers a way to get the fully qualified version as configured by the build.
      * This will be the same as {@link Version} for production releases, but may include on of the qualifier ( e.x alpha1 )
      * or -SNAPSHOT for others.
diff --git a/libs/core/src/main/java/org/opensearch/LegacyESVersion.java b/libs/core/src/main/java/org/opensearch/LegacyESVersion.java
index 32eae654cf975..5d8e067a8fd8b 100644
--- a/libs/core/src/main/java/org/opensearch/LegacyESVersion.java
+++ b/libs/core/src/main/java/org/opensearch/LegacyESVersion.java
@@ -40,7 +40,7 @@
 
 /**
  * The Contents of this file were originally moved from {@link Version}.
- *
+ * 
    
  * This class keeps all the supported OpenSearch predecessor versions for
  * backward compatibility purpose.
  *
diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchException.java b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
index 5bad711a15032..cce86b452f698 100644
--- a/libs/core/src/main/java/org/opensearch/OpenSearchException.java
+++ b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
@@ -168,7 +168,7 @@ public OpenSearchException(Throwable cause) {
 
     /**
      * Construct a OpenSearchException with the specified detail message.
-     *
+     * 
    
      * The message can be parameterized using {} as placeholders for the given
      * arguments
      *
@@ -182,7 +182,7 @@ public OpenSearchException(String msg, Object... args) {
     /**
      * Construct a OpenSearchException with the specified detail message
      * and nested exception.
-     *
+     * 
    
      * The message can be parameterized using {} as placeholders for the given
      * arguments
      *
@@ -587,7 +587,7 @@ public static OpenSearchException innerFromXContent(XContentParser parser, boole
      * Static toXContent helper method that renders {@link OpenSearchException} or {@link Throwable} instances
      * as XContent, delegating the rendering to {@link OpenSearchException#toXContent(XContentBuilder, ToXContent.Params)}
      * or {@link #innerToXContent(XContentBuilder, ToXContent.Params, Throwable, String, String, Map, Map, Throwable)}.
-     *
+     * 
    
      * This method is usually used when the {@link Throwable} is rendered as a part of another XContent object, and its result can
      * be parsed back using the {@code OpenSearchException.fromXContent(XContentParser)} method.
      */
@@ -606,7 +606,7 @@ public static void generateThrowableXContent(XContentBuilder builder, ToXContent
      * depends on the value of the "detailed" parameter: when it's false only a simple message based on the type and message of the
      * exception is rendered. When it's true all detail are provided including guesses root causes, cause and potentially stack
      * trace.
-     *
+     * 
    
      * This method is usually used when the {@link Exception} is rendered as a full XContent object, and its output can be parsed
      * by the {@code #OpenSearchException.failureFromXContent(XContentParser)} method.
      */
diff --git a/libs/core/src/main/java/org/opensearch/core/action/ActionListener.java b/libs/core/src/main/java/org/opensearch/core/action/ActionListener.java
index 119e56cfe0bf2..4fd55898a2cb5 100644
--- a/libs/core/src/main/java/org/opensearch/core/action/ActionListener.java
+++ b/libs/core/src/main/java/org/opensearch/core/action/ActionListener.java
@@ -154,9 +154,9 @@ static  ActionListener wrap(Runnable runnable) {
     /**
      * Creates a listener that wraps another listener, mapping response values via the given mapping function and passing along
      * exceptions to the delegate.
-     *
+     * 
    
      * Notice that it is considered a bug if the listener's onResponse or onFailure fails. onResponse failures will not call onFailure.
-     *
+     * 
    
      * If the function fails, the listener's onFailure handler will be called. The principle is that the mapped listener will handle
      * exceptions from the mapping function {@code fn} but it is the responsibility of {@code delegate} to handle its own exceptions
      * inside `onResponse` and `onFailure`.
@@ -334,7 +334,7 @@ protected void innerOnFailure(Exception e) {
     /**
      * Completes the given listener with the result from the provided supplier accordingly.
      * This method is mainly used to complete a listener with a block of synchronous code.
-     *
+     * 
    
      * If the supplier fails, the listener's onFailure handler will be called.
      * It is the responsibility of {@code delegate} to handle its own exceptions inside `onResponse` and `onFailure`.
      */
diff --git a/libs/core/src/main/java/org/opensearch/core/common/Strings.java b/libs/core/src/main/java/org/opensearch/core/common/Strings.java
index 6227716af9cc9..8fdec670bd9f2 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/Strings.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/Strings.java
@@ -38,7 +38,7 @@
 
 /**
  * String utility class.
- *
+ * 
    
  * TODO replace Strings in :server
  *
  * @opensearch.internal
diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/CompositeBytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/CompositeBytesReference.java
index 53915a3da824c..1a48abee2dbf8 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/CompositeBytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/CompositeBytesReference.java
@@ -45,7 +45,7 @@
 /**
  * A composite {@link BytesReference} that allows joining multiple bytes references
  * into one without copying.
- *
+ * 
    
  * Note, {@link #toBytesRef()} will materialize all pages in this BytesReference.
  *
  * @opensearch.internal
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java
index a50d1c165ed72..30c84708728ef 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java
@@ -17,7 +17,7 @@
  * {@link StreamInput} version of Lucene's {@link org.apache.lucene.store.ByteArrayDataInput}
  * This is used as a replacement of Lucene ByteArrayDataInput for abstracting byte order changes
  * in Lucene's API
- *
+ * 
    
  * Attribution given to apache lucene project under ALv2:
  *
  * Licensed to the Apache Software Foundation (ASF) under one or more
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java
index abac76c8b6c27..123b52eb92876 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/NamedWriteableRegistry.java
@@ -43,7 +43,7 @@
 
 /**
  * A registry for {@link Writeable.Reader} readers of {@link NamedWriteable}.
- *
+ * 
    
  * The registration is keyed by the combination of the category class of {@link NamedWriteable}, and a name unique
  * to that category.
  *
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
index ece2012302919..3e996bdee83a2 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
@@ -96,7 +96,7 @@
 
 /**
  * A stream from this node to another node. Technically, it can also be streamed to a byte array but that is mostly for testing.
- *
+ * 
    
  * This class's methods are optimized so you can put the methods that read and write a class next to each other and you can scan them
  * visually for differences. That means that most variables should be read and written in a single line so even large objects fit both
  * reading and writing on the screen. It also means that the methods on this class are named very similarly to {@link StreamOutput}. Finally
@@ -1128,7 +1128,7 @@ public  C readNamedWriteable(@SuppressWarnings("unused
      * the corresponding entry in the registry by name, so that the proper object can be read and returned.
      * Default implementation throws {@link UnsupportedOperationException} as StreamInput doesn't hold a registry.
      * Use {@link FilterInputStream} instead which wraps a stream and supports a {@link NamedWriteableRegistry} too.
-     *
+     * 
    
      * Prefer {@link StreamInput#readNamedWriteable(Class)} and {@link StreamOutput#writeNamedWriteable(NamedWriteable)} unless you
      * have a compelling reason to use this method instead.
      */
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java
index 94b813246bc7e..2d69e1c686df3 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java
@@ -88,7 +88,7 @@
 
 /**
  * A stream from another node to this node. Technically, it can also be streamed from a byte array but that is mostly for testing.
- *
+ * 
    
  * This class's methods are optimized so you can put the methods that read and write a class next to each other and you can scan them
  * visually for differences. That means that most variables should be read and written in a single line so even large objects fit both
  * reading and writing on the screen. It also means that the methods on this class are named very similarly to {@link StreamInput}. Finally
diff --git a/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java b/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java
index 322300a554284..45ee72f558724 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/settings/SecureString.java
@@ -50,7 +50,7 @@ public final class SecureString implements CharSequence, Closeable {
 
     /**
      * Constructs a new SecureString which controls the passed in char array.
-     *
+     * 
    
      * Note: When this instance is closed, the array will be zeroed out.
      */
     public SecureString(char[] chars) {
@@ -59,7 +59,7 @@ public SecureString(char[] chars) {
 
     /**
      * Constructs a new SecureString from an existing String.
-     *
+     * 
    
      * NOTE: This is not actually secure, since the provided String cannot be deallocated, but
      * this constructor allows for easy compatibility between new and old apis.
      *
diff --git a/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java
index c15db75d06d49..49eadbbb2bc00 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java
@@ -45,7 +45,7 @@
  * A {@code SizeUnit} does not maintain size information, but only
  * helps organize and use size representations that may be maintained
  * separately across various contexts.
- *
+ * 
    
  * It use conventional data storage values (base-2) :
  * 
      
  *     
    • 1KB = 1024 bytes
      • 
diff --git a/libs/core/src/main/java/org/opensearch/core/compress/Compressor.java b/libs/core/src/main/java/org/opensearch/core/compress/Compressor.java
index 27d5b5dfdfa15..5324ea6151e51 100644
--- a/libs/core/src/main/java/org/opensearch/core/compress/Compressor.java
+++ b/libs/core/src/main/java/org/opensearch/core/compress/Compressor.java
@@ -43,7 +43,7 @@
 /**
  * Compressor interface used for compressing {@link org.opensearch.core.xcontent.MediaType} and
  * {@code org.opensearch.repositories.blobstore.BlobStoreRepository} implementations.
- *
+ * 
      
  * This is not to be confused with {@link org.apache.lucene.codecs.compressing.Compressor} which is used
  * for codec implementations such as {@code org.opensearch.index.codec.customcodecs.Lucene95CustomCodec}
  * for compressing {@link org.apache.lucene.document.StoredField}s
diff --git a/libs/core/src/main/java/org/opensearch/core/compress/CompressorRegistry.java b/libs/core/src/main/java/org/opensearch/core/compress/CompressorRegistry.java
index 9290254c30d8d..af09a7aebba79 100644
--- a/libs/core/src/main/java/org/opensearch/core/compress/CompressorRegistry.java
+++ b/libs/core/src/main/java/org/opensearch/core/compress/CompressorRegistry.java
@@ -23,7 +23,7 @@
 /**
  * A registry that wraps a static Map singleton which holds a mapping of unique String names (typically the
  * compressor header as a string) to registerd {@link Compressor} implementations.
- *
+ * 
      
  * This enables plugins, modules, extensions to register their own compression implementations through SPI
  *
  * @opensearch.experimental
@@ -105,7 +105,7 @@ public static Compressor getCompressor(final String name) {
 
     /**
      * Returns the registered compressors as an Immutable collection
-     *
+     * 
      
      * note: used for testing
      */
     public static Map registeredCompressors() {
diff --git a/libs/core/src/main/java/org/opensearch/core/compress/spi/CompressorProvider.java b/libs/core/src/main/java/org/opensearch/core/compress/spi/CompressorProvider.java
index 019e282444d64..9b806618fe0a0 100644
--- a/libs/core/src/main/java/org/opensearch/core/compress/spi/CompressorProvider.java
+++ b/libs/core/src/main/java/org/opensearch/core/compress/spi/CompressorProvider.java
@@ -18,7 +18,7 @@
 /**
  * Service Provider Interface for plugins, modules, extensions providing custom
  * compression algorithms
- *
+ * 
      
  * see {@link Compressor} for implementing methods
  * and {@link org.opensearch.core.compress.CompressorRegistry} for the registration of custom
  * Compressors
diff --git a/libs/core/src/main/java/org/opensearch/core/index/Index.java b/libs/core/src/main/java/org/opensearch/core/index/Index.java
index b5ee482bcd952..a927179114188 100644
--- a/libs/core/src/main/java/org/opensearch/core/index/Index.java
+++ b/libs/core/src/main/java/org/opensearch/core/index/Index.java
@@ -48,7 +48,7 @@
 
 /**
  * A value class representing the basic required properties of an OpenSearch index.
- *
+ * 
      
  * (This class is immutable.)
  *
  * @opensearch.api
@@ -116,7 +116,7 @@ public String getUUID() {
     /**
      * Returns either the name and unique identifier of the index
      * or only the name if the uuid is {@link Strings#UNKNOWN_UUID_VALUE}.
-     *
+     * 
      
      * If we have a uuid we put it in the toString so it'll show up in logs
      * which is useful as more and more things use the uuid rather
      * than the name as the lookup key for the index.
diff --git a/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java b/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java
index 984434190b486..c0abad7ed727f 100644
--- a/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java
+++ b/libs/core/src/main/java/org/opensearch/core/index/shard/ShardId.java
@@ -143,7 +143,7 @@ public String toString() {
 
     /**
      * Parse the string representation of this shardId back to an object.
-     *
+     * 
      
      * We lose index uuid information here, but since we use toString in
      * rest responses, this is the best we can do to reconstruct the object
      * on the client side.
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java
index a0e2a54fce91c..79e531a542026 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java
@@ -108,7 +108,7 @@ public abstract  void declareNamedObject(
      *
      * Unlike the other version of this method, "ordered" mode (arrays of
      * objects) is not supported.
-     *
+     * 
      
      * See NamedObjectHolder in ObjectParserTests for examples of how to invoke
      * this.
      *
@@ -163,7 +163,7 @@ public abstract  void declareNamedObjects(
      * the order sent but tools that generate json are free to put object
      * members in an unordered Map, jumbling them. Thus, if you care about order
      * you can send the object in the second way.
-     *
+     * 
      
      * See NamedObjectHolder in ObjectParserTests for examples of how to invoke
      * this.
      *
@@ -366,10 +366,10 @@ public  void declareFieldArray(
     /**
      * Declares a set of fields that are required for parsing to succeed. Only one of the values
      * provided per String[] must be matched.
-     *
+     * 
      
      * E.g. declareRequiredFieldSet("foo", "bar"); means at least one of "foo" or
      * "bar" fields must be present.  If neither of those fields are present, an exception will be thrown.
-     *
+     * 
      
      * Multiple required sets can be configured:
      *
      * 
      
@@ -379,7 +379,7 @@ public  void declareFieldArray(
      *
      * requires that one of "foo" or "bar" fields are present, and also that one of "bizz" or
      * "buzz" fields are present.
-     *
+     * 
      
      * In JSON, it means any of these combinations are acceptable:
      *
      * 
        
@@ -415,12 +415,12 @@ public  void declareFieldArray(
 
     /**
      * Declares a set of fields of which at most one must appear for parsing to succeed
-     *
+     * 
        
      * E.g. declareExclusiveFieldSet("foo", "bar"); means that only one of 'foo'
      * or 'bar' must be present, and if both appear then an exception will be thrown.  Note
      * that this does not make 'foo' or 'bar' required - see {@link #declareRequiredFieldSet(String...)}
      * for required fields.
-     *
+     * 
        
      * Multiple exclusive sets may be declared
      *
      * @param exclusiveSet a set of field names, at most one of which must appear
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MapXContentParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MapXContentParser.java
index 254c340f8836f..0a5cda324ddb7 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/MapXContentParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MapXContentParser.java
@@ -277,7 +277,7 @@ public Token currentToken() {
 
         /**
          * field name that the child element needs to inherit.
-         *
+         * 
        
          * In most cases this is the same as currentName() except with embedded arrays. In "foo": [[42]] the first START_ARRAY
          * token will have the name "foo", but the second START_ARRAY will have no name.
          */
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilder.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilder.java
index a38bdd049ee88..976f353100c55 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilder.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilder.java
@@ -726,7 +726,7 @@ public XContentBuilder value(byte[] value, int offset, int length) throws IOExce
 
     /**
      * Writes the binary content of the given byte array as UTF-8 bytes.
-     *
+     * 
        
      * Use {@link XContentParser#charBuffer()} to read the value back
      */
     public XContentBuilder utf8Value(byte[] bytes, int offset, int length) throws IOException {
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilderExtension.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilderExtension.java
index 0535da1a584be..9b13ebb23be86 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilderExtension.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentBuilderExtension.java
@@ -37,7 +37,7 @@
 
 /**
  * This interface provides a way for non-JDK classes to plug in a way to serialize to xcontent.
- *
+ * 
        
  * It is greatly preferred that you implement {@link ToXContentFragment}
  * in the class for encoding, however, in some situations you may not own the
  * class, in which case you can add an implementation here for encoding it.
@@ -63,7 +63,7 @@ public interface XContentBuilderExtension {
      * Used for plugging in a human readable version of a class's encoding. It is assumed that
      * the human readable equivalent is always behind the {@code toString()} method, so
      * this transformer returns the raw value to be used.
-     *
+     * 
        
      * An example implementation:
      *
      * 
        @@ -79,7 +79,7 @@ public interface XContentBuilderExtension {
     /**
      * Used for plugging a transformer for a date or time type object into a String (or other
      * encodable object).
-     *
+     * 
        
      * For example:
      *
      * 
        diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java
index a2f16209a5b7f..4bfd47ccfdc94 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java
@@ -44,7 +44,7 @@
 
 /**
  * Interface for pull - parsing {@link XContent} see {@code XContentType} for supported types.
- *
+ * 
        
  * To obtain an instance of this class use the following pattern:
  *
  * 
        @@ -202,11 +202,11 @@  Map map(Supplier> mapFactory, CheckedFunction
      * Default implementation simply returns false since only actual
      * implementation class has knowledge of its internal buffering
      * state.
-     *
+     * 
        
      * This method shouldn't be used to check if the token contains text or not.
      */
     boolean hasTextCharacters();
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java
index 13e2f6a695d1b..b10be393f9adb 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java
@@ -142,10 +142,10 @@ public static Object parseFieldsValue(XContentParser parser) throws IOException
      * This method expects that the current field name is the concatenation of a type, a delimiter and a name
      * (ex: terms#foo where "terms" refers to the type of a registered {@link NamedXContentRegistry.Entry},
      * "#" is the delimiter and "foo" the name of the object to parse).
-     *
+     * 
        
      * It also expected that following this field name is either an Object or an array xContent structure and
      * the cursor points to the start token of this structure.
-     *
+     * 
        
      * The method splits the field's name to extract the type and name and then parses the object
      * using the {@link XContentParser#namedObject(Class, String, Object)} method.
      *
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentSubParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentSubParser.java
index d1cdda4aeb8be..337cf9f95fe5f 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentSubParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentSubParser.java
@@ -43,7 +43,7 @@
 
 /**
  * Wrapper for a XContentParser that makes a single object/array look like a complete document.
- *
+ * 
        
  * The wrapper prevents the parsing logic to consume tokens outside of the wrapped object as well
  * as skipping to the end of the object in case of a parsing error. The wrapper is intended to be
  * used for parsing objects that should be ignored if they are malformed.
diff --git a/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java
index 9861847c9e1ea..b6dc0ceb1028f 100644
--- a/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java
+++ b/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java
@@ -194,26 +194,25 @@ public DissectParser(String pattern, String appendSeparator) {
      * @throws DissectException if unable to dissect a pair into it's parts.
      */
     public Map parse(String inputString) {
-        /**
-         *
-         * This implements a naive string matching algorithm. The string is walked left to right, comparing each byte against
-         * another string's bytes looking for matches. If the bytes match, then a second cursor looks ahead to see if all the bytes
-         * of the other string matches. If they all match, record it and advances the primary cursor to the match point. If it can not match
-         * all of the bytes then progress the main cursor. Repeat till the end of the input string. Since the string being searching for
-         * (the delimiter) is generally small and rare the naive approach is efficient.
-         *
-         * In this case the string that is walked is the input string, and the string being searched for is the current delimiter.
-         * For example for a dissect pattern of {@code %{a},%{b}:%{c}} the delimiters (comma then colon) are searched for in the
-         * input string. At class construction the list of keys+delimiters are found (dissectPairs), which allows the use of that ordered
-         * list to know which delimiter to use for the search. The delimiters is progressed once the current delimiter is matched.
-         *
-         * There are two special cases that requires additional parsing beyond the standard naive algorithm. Consecutive delimiters should
-         * results in a empty matches unless the {@code ->} is provided. For example given the dissect pattern of
-         * {@code %{a},%{b},%{c},%{d}} and input string of {@code foo,,,} the match should be successful with empty values for b,c and d.
-         * However, if the key modifier {@code ->}, is present it will simply skip over any delimiters just to the right of the key
-         * without assigning any values. For example {@code %{a->},{%b}} will match the input string of {@code foo,,,,,,bar} with a=foo and
-         * b=bar.
-         *
+        /*
+
+          This implements a naive string matching algorithm. The string is walked left to right, comparing each byte against
+          another string's bytes looking for matches. If the bytes match, then a second cursor looks ahead to see if all the bytes
+          of the other string matches. If they all match, record it and advances the primary cursor to the match point. If it can not match
+          all of the bytes then progress the main cursor. Repeat till the end of the input string. Since the string being searching for
+          (the delimiter) is generally small and rare the naive approach is efficient.
+
+          In this case the string that is walked is the input string, and the string being searched for is the current delimiter.
+          For example for a dissect pattern of {@code %{a},%{b}:%{c}} the delimiters (comma then colon) are searched for in the
+          input string. At class construction the list of keys+delimiters are found (dissectPairs), which allows the use of that ordered
+          list to know which delimiter to use for the search. The delimiters is progressed once the current delimiter is matched.
+
+          There are two special cases that requires additional parsing beyond the standard naive algorithm. Consecutive delimiters should
+          results in a empty matches unless the {@code ->} is provided. For example given the dissect pattern of
+          {@code %{a},%{b},%{c},%{d}} and input string of {@code foo,,,} the match should be successful with empty values for b,c and d.
+          However, if the key modifier {@code ->}, is present it will simply skip over any delimiters just to the right of the key
+          without assigning any values. For example {@code %{a->},{%b}} will match the input string of {@code foo,,,,,,bar} with a=foo and
+          b=bar.
          */
         DissectMatch dissectMatch = new DissectMatch(appendSeparator, maxMatches, maxResults, appendCount, referenceCount);
         Iterator it = matchPairs.iterator();
diff --git a/libs/geo/src/main/java/org/opensearch/geometry/Circle.java b/libs/geo/src/main/java/org/opensearch/geometry/Circle.java
index cf1dce8966e1f..c05f316b53b9c 100644
--- a/libs/geo/src/main/java/org/opensearch/geometry/Circle.java
+++ b/libs/geo/src/main/java/org/opensearch/geometry/Circle.java
@@ -180,6 +180,7 @@ public int hashCode() {
 
     /**
      * Visit this circle with a {@link GeometryVisitor}.
+     *
      * @param visitor The visitor
      * @param  The return type of the visitor
      * @param  The exception type of the visitor
diff --git a/libs/geo/src/main/java/org/opensearch/geometry/utils/BitUtil.java b/libs/geo/src/main/java/org/opensearch/geometry/utils/BitUtil.java
index 664e7e68d96a5..c946cc2473202 100644
--- a/libs/geo/src/main/java/org/opensearch/geometry/utils/BitUtil.java
+++ b/libs/geo/src/main/java/org/opensearch/geometry/utils/BitUtil.java
@@ -48,8 +48,8 @@ public class BitUtil {  // magic numbers for bit interleaving
 
     /**
      * Interleaves the first 32 bits of each long value
-     *
-     * Adapted from: http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
+     * 
        
+     * Adapted from: bithacks.html#InterleaveBMN
      */
     public static long interleave(int even, int odd) {
         long v1 = 0x00000000FFFFFFFFL & even;
diff --git a/libs/geo/src/main/java/org/opensearch/geometry/utils/Geohash.java b/libs/geo/src/main/java/org/opensearch/geometry/utils/Geohash.java
index 8b3b841e221e5..33c423e136613 100644
--- a/libs/geo/src/main/java/org/opensearch/geometry/utils/Geohash.java
+++ b/libs/geo/src/main/java/org/opensearch/geometry/utils/Geohash.java
@@ -39,12 +39,12 @@
 
 /**
  * Utilities for converting to/from the GeoHash standard
- *
+ * 
        
  * The geohash long format is represented as lon/lat (x/y) interleaved with the 4 least significant bits
  * representing the level (1-12) [xyxy...xyxyllll]
- *
+ * 
        
  * This differs from a morton encoded value which interleaves lat/lon (y/x).
- *
+ * 
        
  * NOTE: this will replace {@code org.opensearch.common.geo.GeoHashUtils}
  */
 public class Geohash {
diff --git a/libs/grok/src/main/java/org/opensearch/grok/Grok.java b/libs/grok/src/main/java/org/opensearch/grok/Grok.java
index cd786b74be039..7aa3347ba4f4b 100644
--- a/libs/grok/src/main/java/org/opensearch/grok/Grok.java
+++ b/libs/grok/src/main/java/org/opensearch/grok/Grok.java
@@ -151,7 +151,7 @@ private void validatePatternBank() {
     /**
      * Checks whether patterns reference each other in a circular manner and, if so, fail with an exception.
      * Also checks for malformed pattern definitions and fails with an exception.
-     *
+     * 
        
      * In a pattern, anything between %{ and } or : is considered
      * a reference to another named pattern. This method will navigate to all these named patterns and
      * check for a circular reference.
diff --git a/libs/grok/src/main/java/org/opensearch/grok/MatcherWatchdog.java b/libs/grok/src/main/java/org/opensearch/grok/MatcherWatchdog.java
index 5c7eaca2a634a..d5b7566ecc90f 100644
--- a/libs/grok/src/main/java/org/opensearch/grok/MatcherWatchdog.java
+++ b/libs/grok/src/main/java/org/opensearch/grok/MatcherWatchdog.java
@@ -44,7 +44,7 @@
  * Protects against long running operations that happen between the register and unregister invocations.
  * Threads that invoke {@link #register(Matcher)}, but take too long to invoke the {@link #unregister(Matcher)} method
  * will be interrupted.
- *
+ * 
        
  * This is needed for Joni's {@link org.joni.Matcher#search(int, int, int)} method, because
  * it can end up spinning endlessly if the regular expression is too complex. Joni has checks
  * that for every 30k iterations it checks if the current thread is interrupted and if so
diff --git a/libs/nio/src/main/java/org/opensearch/nio/ChannelContext.java b/libs/nio/src/main/java/org/opensearch/nio/ChannelContext.java
index 797dfe859fa6c..0e29661978716 100644
--- a/libs/nio/src/main/java/org/opensearch/nio/ChannelContext.java
+++ b/libs/nio/src/main/java/org/opensearch/nio/ChannelContext.java
@@ -116,10 +116,10 @@ protected void handleException(Exception e) {
 
     /**
      * Schedules a channel to be closed by the selector event loop with which it is registered.
-     *
+     * 
        
      * If the channel is open and the state can be transitioned to closed, the close operation will
      * be scheduled with the event loop.
-     *
+     * 
        
      * Depending on the underlying protocol of the channel, a close operation might simply close the socket
      * channel or may involve reading and writing messages.
      */
diff --git a/libs/nio/src/main/java/org/opensearch/nio/NioSelector.java b/libs/nio/src/main/java/org/opensearch/nio/NioSelector.java
index a38a33182afea..4ed745723515c 100644
--- a/libs/nio/src/main/java/org/opensearch/nio/NioSelector.java
+++ b/libs/nio/src/main/java/org/opensearch/nio/NioSelector.java
@@ -512,12 +512,12 @@ private void handleQueuedWrites() {
      * This is a convenience method to be called after some object (normally channels) are enqueued with this
      * selector. This method will check if the selector is still open. If it is open, normal operation can
      * proceed.
-     *
+     * 
        
      * If the selector is closed, then we attempt to remove the object from the queue. If the removal
      * succeeds then we throw an {@link IllegalStateException} indicating that normal operation failed. If
      * the object cannot be removed from the queue, then the object has already been handled by the selector
      * and operation can proceed normally.
-     *
+     * 
        
      * If this method is called from the selector thread, we will not allow the queuing to occur as the
      * selector thread can manipulate its queues internally even if it is no longer open.
      *
diff --git a/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java b/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java
index 12a1e80055823..3df8e42fe4f14 100644
--- a/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java
+++ b/libs/nio/src/main/java/org/opensearch/nio/SocketChannelContext.java
@@ -59,7 +59,7 @@
  * that it is ready to perform certain operations (read, write, etc) the {@link SocketChannelContext} will
  * be called. This context will need to implement all protocol related logic. Additionally, if any special
  * close behavior is required, it should be implemented in this context.
- *
+ * 
        
  * The only methods of the context that should ever be called from a non-selector thread are
  * {@link #closeChannel()} and {@link #sendMessage(Object, BiConsumer)}.
  */
diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java
index 49295f417df9e..e6d4878a5e833 100644
--- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java
+++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java
@@ -16,7 +16,7 @@
 /**
  * Tracer is the interface used to create a {@link Span}
  * It automatically handles the context propagation between threads, tasks, nodes etc.
- *
+ * 
        
  * All methods on the Tracer object are multi-thread safe.
  *
  * @opensearch.experimental
diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java
index b0692f1b62a48..50d18c0a0d040 100644
--- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java
+++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/http/HttpTracer.java
@@ -18,7 +18,7 @@
 /**
  * HttpTracer helps in creating a {@link Span} which reads the incoming tracing information
  * from the HttpRequest header and propagate the span accordingly.
- *
+ * 
        
  * All methods on the Tracer object are multi-thread safe.
  *
  * @opensearch.experimental
diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
index 3b857a2c690b9..d3d9ea174cf1b 100644
--- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
+++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
@@ -628,7 +628,7 @@ public void testCreateRootSubParser() throws IOException {
 
     /**
      * Generates a random object {"first_field": "foo", "marked_field": {...random...}, "last_field": "bar}
-     *
+     * 
        
      * Returns the number of tokens in the marked field
      */
     private static int generateRandomObjectForMarking(XContentBuilder builder) throws IOException {
diff --git a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java
index de67cc2930652..de6b59b1546a5 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/opensearch/search/aggregations/matrix/stats/RunningStats.java
@@ -46,7 +46,7 @@
 /**
  * Descriptive stats gathered per shard. Coordinating node computes final correlation and covariance stats
  * based on these descriptive stats. This single pass, parallel approach is based on:
- *
+ * 
        
  * http://prod.sandia.gov/techlib/access-control.cgi/2008/086212.pdf
  */
 public class RunningStats implements Writeable, Cloneable {
@@ -222,7 +222,7 @@ private void updateCovariance(final String[] fieldNames, final Map
      * running computations taken from: http://prod.sandia.gov/techlib/access-control.cgi/2008/086212.pdf
      **/
     public void merge(final RunningStats other) {
diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java
index b0d9c1765190a..edb8c37c2dbdd 100644
--- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java
+++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java
@@ -555,7 +555,7 @@ public List getPreConfiguredTokenFilters() {
         filters.add(PreConfiguredTokenFilter.singleton("scandinavian_normalization", true, ScandinavianNormalizationFilter::new));
         filters.add(PreConfiguredTokenFilter.singleton("shingle", false, false, input -> {
             TokenStream ts = new ShingleFilter(input);
-            /**
+            /*
              * We disable the graph analysis on this token stream
              * because it produces shingles of different size.
              * Graph analysis on such token stream is useless and dangerous as it may create too many paths
diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/KeywordMarkerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/KeywordMarkerTokenFilterFactory.java
index ad968aeee62cb..e9f3fd96dd69d 100644
--- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/KeywordMarkerTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/KeywordMarkerTokenFilterFactory.java
@@ -49,12 +49,12 @@
  * A factory for creating keyword marker token filters that prevent tokens from
  * being modified by stemmers.  Two types of keyword marker filters are available:
  * the {@link SetKeywordMarkerFilter} and the {@link PatternKeywordMarkerFilter}.
- *
+ * 
        
  * The {@link SetKeywordMarkerFilter} uses a set of keywords to denote which tokens
  * should be excluded from stemming.  This filter is created if the settings include
  * {@code keywords}, which contains the list of keywords, or {@code `keywords_path`},
  * which contains a path to a file in the config directory with the keywords.
- *
+ * 
        
  * The {@link PatternKeywordMarkerFilter} uses a regular expression pattern to match
  * against tokens that should be excluded from stemming.  This filter is created if
  * the settings include {@code keywords_pattern}, which contains the regular expression
diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SnowballAnalyzer.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SnowballAnalyzer.java
index 78d151ee16c3b..04786689b50f0 100644
--- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SnowballAnalyzer.java
+++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SnowballAnalyzer.java
@@ -45,7 +45,7 @@
 
 /** Filters {@link StandardTokenizer} with {@link
  * LowerCaseFilter}, {@link StopFilter} and {@link SnowballFilter}.
- *
+ * 
        
  * Available stemmers are listed in org.tartarus.snowball.ext.  The name of a
  * stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
  * {@link org.tartarus.snowball.ext.EnglishStemmer} is named "English".
diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java
index 9149b8939b739..665ea6c5f2f37 100644
--- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java
+++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java
@@ -48,7 +48,7 @@
 
 /**
  * A {@link SingleDimensionValuesSource} for geotile values.
- *
+ * 
        
  * Since geotile values can be represented as long values, this class is almost the same as {@link LongValuesSource}
  * The main differences is {@link GeoTileValuesSource#setAfter(Comparable)} as it needs to accept geotile string values i.e. "zoom/x/y".
  *
diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/BoundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/BoundedCellValues.java
index 588c8bc59c2e0..6ff38fa28978e 100644
--- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/BoundedCellValues.java
+++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/BoundedCellValues.java
@@ -37,7 +37,7 @@
 /**
  * Class representing {@link CellValues} whose values are filtered
  * according to whether they are within the specified {@link GeoBoundingBox}.
- *
+ * 
        
  * The specified bounding box is assumed to be bounded.
  *
  * @opensearch.internal
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java
index 37320c0e900a5..7e114023fb86f 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/FailProcessorException.java
@@ -37,7 +37,7 @@
 
 /**
  * Exception class thrown by {@link FailProcessor}.
- *
+ * 
        
  * This exception is caught in the {@link CompoundProcessor} and
  * then changes the state of {@link IngestDocument}. This
  * exception should get serialized.
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java
index 741a4fb29cfb8..b7c417f5f44a5 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ForEachProcessor.java
@@ -53,10 +53,10 @@
 
 /**
  * A processor that for each value in a list executes a one or more processors.
- *
+ * 
        
  * This can be useful in cases to do string operations on json array of strings,
  * or remove a field from objects inside a json array.
- *
+ * 
        
  * Note that this processor is experimental.
  */
 public final class ForEachProcessor extends AbstractProcessor implements WrappingProcessor {
diff --git a/modules/lang-expression/src/main/java/org/opensearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/opensearch/script/expression/ExpressionScriptEngine.java
index 035d2402857e0..11b2e20eea523 100644
--- a/modules/lang-expression/src/main/java/org/opensearch/script/expression/ExpressionScriptEngine.java
+++ b/modules/lang-expression/src/main/java/org/opensearch/script/expression/ExpressionScriptEngine.java
@@ -73,7 +73,7 @@
 
 /**
  * Provides the infrastructure for Lucene expressions as a scripting language for OpenSearch.
- *
+ * 
        
  * Only contexts returning numeric types or {@link Object} are supported.
  */
 public class ExpressionScriptEngine implements ScriptEngine {
diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MustacheScriptEngine.java
index f4d7198dc2124..ec84475b70bb6 100644
--- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MustacheScriptEngine.java
+++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MustacheScriptEngine.java
@@ -59,7 +59,7 @@
 /**
  * Main entry point handling template registration, compilation and
  * execution.
- *
+ * 
        
  * Template handling is based on Mustache. Template handling is a two step
  * process: First compile the string representing the template, the resulting
  * {@link Mustache} object can then be re-used for subsequent executions.
diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheScriptEngineTests.java
index 9e97863306148..fbb7d09709a91 100644
--- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheScriptEngineTests.java
+++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MustacheScriptEngineTests.java
@@ -200,7 +200,7 @@ private String getChars() {
 
     /**
      * From https://www.ietf.org/rfc/rfc4627.txt:
-     *
+     * 
        
      * All Unicode characters may be placed within the
      * quotation marks except for the characters that must be escaped:
      * quotation mark, reverse solidus, and the control characters (U+0000
diff --git a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/Allowlist.java b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/Allowlist.java
index 56ade63efa5e2..265263b41ca89 100644
--- a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/Allowlist.java
+++ b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/Allowlist.java
@@ -42,7 +42,7 @@
  * Allowlist contains data structures designed to be used to generate an allowlist of Java classes,
  * constructors, methods, and fields that can be used within a Painless script at both compile-time
  * and run-time.
- *
+ * 
        
  * A Allowlist consists of several pieces with {@link AllowlistClass}s as the top level. Each
  * {@link AllowlistClass} will contain zero-to-many {@link AllowlistConstructor}s, {@link AllowlistMethod}s, and
  * {@link AllowlistField}s which are what will be available with a Painless script.  See each individual
diff --git a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistClass.java b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistClass.java
index 17e6814addf3b..67f5a07846c53 100644
--- a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistClass.java
+++ b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistClass.java
@@ -45,12 +45,12 @@
  * classes. Though, since multiple allowlists may be combined into a single allowlist for a
  * specific context, as long as multiple classes representing the same Java class have the same
  * class name and have legal constructor/method overloading they can be merged together.
- *
+ * 
        
  * Classes in Painless allow for arity overloading for constructors and methods. Arity overloading
  * means that multiple constructors are allowed for a single class as long as they have a different
  * number of parameters, and multiples methods with the same name are allowed for a single class
  * as long as they have the same return type and a different number of parameters.
- *
+ * 
        
  * Classes will automatically extend other allowlisted classes if the Java class they represent is a
  * subclass of other classes including Java interfaces.
  */
diff --git a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java
index 71265f82acacc..632fee9187eba 100644
--- a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java
+++ b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java
@@ -67,11 +67,11 @@ public static Allowlist loadFromResourceFiles(Class resource, String... filep
      * is the path of a single text file. The {@link Class}'s {@link ClassLoader} will be used to lookup the Java
      * reflection objects for each individual {@link Class}, {@link Constructor}, {@link Method}, and {@link Field}
      * specified as part of the allowlist in the text file.
-     *
+     * 
        
      * A single pass is made through each file to collect all the information about each class, constructor, method,
      * and field. Most validation will be done at a later point after all allowlists have been gathered and their
      * merging takes place.
-     *
+     * 
        
      * A painless type name is one of the following:
      * 
          
      *     
        •  def - The Painless dynamic type which is automatically included without a need to be
@@ -129,13 +129,13 @@ public static Allowlist loadFromResourceFiles(Class resource, String... filep
      * be appropriately parsed and handled. Painless complex types must be specified with the
      * fully-qualified Java class name. Method argument types, method return types, and field types
      * must be specified with Painless type names (def, fully-qualified, or short) as described earlier.
-     *
+     * 
          
      * The following example is used to create a single allowlist text file:
      *
-     * {@code
+     * 
                * # primitive types
      *
-     * class int -> int {
+     * class int -> int {
      * }
      *
      * # complex types
@@ -161,7 +161,7 @@ public static Allowlist loadFromResourceFiles(Class resource, String... filep
      *   int value1
      *   def value2
      * }
-     * }
+     * 
          
      */
     public static Allowlist loadFromResourceFiles(Class resource, Map parsers, String... filepaths) {
         List allowlistClasses = new ArrayList<>();
diff --git a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistMethod.java b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistMethod.java
index 8bb0231ff3f4f..9fcaec3dbf7b6 100644
--- a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistMethod.java
+++ b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistMethod.java
@@ -45,7 +45,7 @@
  * are using the '.' operator on an existing class variable/field. Painless classes may have multiple
  * methods with the same name as long as they comply with arity overloading described in
  * {@link AllowlistClass}.
- *
+ * 
          
  * Classes may also have additional methods that are not part of the Java class the class represents -
  * these are known as augmented methods. An augmented method can be added to a class as a part of any
  * Java class as long as the method is static and the first parameter of the method is the Java class
diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java
index 35c676653fdc3..c19d4f361b2b6 100644
--- a/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java
@@ -73,9 +73,7 @@ final class Compiler {
      */
     private static final CodeSource CODESOURCE;
 
-    /**
-     * Setup the code privileges.
-     */
+    /* Setup the code privileges. */
     static {
         try {
             // Setup the code privileges.
diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupUtility.java
index cae425ad1fe3b..3164f5e6388c7 100644
--- a/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupUtility.java
@@ -42,13 +42,13 @@
 /**
  * PainlessLookupUtility contains methods shared by {@link PainlessLookupBuilder}, {@link PainlessLookup}, and other classes within
  * Painless for conversion between type names and types along with some other various utility methods.
- *
+ * 
          
  * The following terminology is used for variable names throughout the lookup package:
- *
+ * 
          
  * A class is a set of methods and fields under a specific class name. A type is either a class or an array under a specific type name.
  * Note the distinction between class versus type is class means that no array classes will be be represented whereas type allows array
  * classes to be represented. The set of available classes will always be a subset of the available types.
- *
+ * 
          
  * Under ambiguous circumstances most variable names are prefixed with asm, java, or painless. If the variable value is the same for asm,
  * java, and painless, no prefix is used. Target is used as a prefix to represent if a constructor, method, or field is being
  * called/accessed on that specific class.  Parameter is often a postfix used to represent if a type is used as a parameter to a
diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/phase/PainlessSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/opensearch/painless/phase/PainlessSemanticAnalysisPhase.java
index 04165f44ba212..8a05d6742af97 100644
--- a/modules/lang-painless/src/main/java/org/opensearch/painless/phase/PainlessSemanticAnalysisPhase.java
+++ b/modules/lang-painless/src/main/java/org/opensearch/painless/phase/PainlessSemanticAnalysisPhase.java
@@ -126,9 +126,9 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
 
     /**
      * Visits an expression that is also considered a statement.
-     *
+     * 
          
      * If the statement is a return from the execute method, performs return value conversion.
-     *
+     * 
          
      * Checks: control flow, type validation
      */
     @Override
@@ -168,9 +168,9 @@ public void visitExpression(SExpression userExpressionNode, SemanticScope semant
 
     /**
      * Visits a return statement and casts the value to the return type if possible.
-     *
+     * 
          
      * If the statement is a return from the execute method, performs return value conversion.
-     *
+     * 
          
      * Checks: type validation
      */
     @Override
diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/symbol/SemanticScope.java b/modules/lang-painless/src/main/java/org/opensearch/painless/symbol/SemanticScope.java
index e27530d745e8f..5ac802038afa6 100644
--- a/modules/lang-painless/src/main/java/org/opensearch/painless/symbol/SemanticScope.java
+++ b/modules/lang-painless/src/main/java/org/opensearch/painless/symbol/SemanticScope.java
@@ -49,7 +49,7 @@
  * Tracks information within a scope required for compilation during the
  * semantic phase in the user tree. There are three types of scopes -
  * {@link FunctionScope}, {@link LambdaScope}, and {@link BlockScope}.
- *
+ * 
          
  * Scopes are stacked as they are created during the user tree's semantic
  * phase with each scope beyond the top-level containing a reference to
  * its parent. As a scope is no longer necessary, it's dropped automatically
diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
index 397a7b48b472a..366e848416328 100644
--- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
@@ -82,7 +82,7 @@
 /**
  * Mapper for a text field that optimizes itself for as-you-type completion by indexing its content into subfields. Each subfield
  * modifies the analysis chain of the root field to index terms the user would create as they type out the value in the root field
- *
+ * 
          
  * The structure of these fields is
  *
  * 
          diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java
index 1a51259e9e4e4..e930780613ed6 100644
--- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java
+++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java
@@ -373,7 +373,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
      * A query that rewrites into another query using
      * {@link JoinUtil#createJoinQuery(String, Query, Query, IndexSearcher, ScoreMode, OrdinalMap, int, int)}
      * that executes the actual join.
-     *
+     * 
          
      * This query is exclusively used by the {@link HasChildQueryBuilder} and {@link HasParentQueryBuilder} to get access
      * to the {@link DirectoryReader} used by the current search in order to retrieve the {@link OrdinalMap}.
      * The {@link OrdinalMap} is required by {@link JoinUtil} to execute the join.
diff --git a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java
index ffdea14e0873d..8e72c6ef06849 100644
--- a/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java
+++ b/modules/rank-eval/src/main/java/org/opensearch/index/rankeval/TransportRankEvalAction.java
@@ -71,10 +71,10 @@
  * supplied query parameters) against a set of possible search requests (read:
  * search specifications, expressed as query/search request templates) and
  * compares the result against a set of annotated documents per search intent.
- *
+ * 
          
  * If any documents are returned that haven't been annotated the document id of
  * those is returned per search intent.
- *
+ * 
          
  * The resulting search quality is computed in terms of precision at n and
  * returned for each search specification for the full set of search intents as
  * averaged precision at n.
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java
index fcf1fa489f740..d96e3212e05a2 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -67,7 +67,7 @@ public class DiscountedCumulativeGainTests extends OpenSearchTestCase {
 
     /**
      * Assuming the docs are ranked in the following order:
-     *
+     * 
          
      * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
      * -------------------------------------------------------------------------------------------
      * 1 | 3 | 7.0 | 1.0 | 7.0 | 7.0 | 
@@ -76,7 +76,7 @@ public class DiscountedCumulativeGainTests extends OpenSearchTestCase {
      * 4 | 0 | 0.0 | 2.321928094887362 | 0.0
      * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163
      * 6 | 2 | 3.0 | 2.807354922057604 | 1.0686215613240666
-     *
+     * 
          
      * dcg = 13.84826362927298 (sum of last column)
      */
     public void testDCGAt() {
@@ -91,20 +91,20 @@ public void testDCGAt() {
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         assertEquals(EXPECTED_DCG, dcg.evaluate("id", hits, rated).metricScore(), DELTA);
 
-        /**
-         * Check with normalization: to get the maximal possible dcg, sort documents by
-         * relevance in descending order
-         *
-         * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
-         * ---------------------------------------------------------------------------------------
-         * 1 | 3 | 7.0 | 1.0  | 7.0
-         * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
-         * 3 | 2 | 3.0 | 2.0  | 1.5
-         * 4 | 2 | 3.0 | 2.321928094887362 | 1.2920296742201793
-         * 5 | 1 | 1.0 | 2.584962500721156  | 0.38685280723454163
-         * 6 | 0 | 0.0 | 2.807354922057604  | 0.0
-         *
-         * idcg = 14.595390756454922 (sum of last column)
+        /*
+          Check with normalization: to get the maximal possible dcg, sort documents by
+          relevance in descending order
+
+          rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
+          ---------------------------------------------------------------------------------------
+          1 | 3 | 7.0 | 1.0  | 7.0
+          2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
+          3 | 2 | 3.0 | 2.0  | 1.5
+          4 | 2 | 3.0 | 2.321928094887362 | 1.2920296742201793
+          5 | 1 | 1.0 | 2.584962500721156  | 0.38685280723454163
+          6 | 0 | 0.0 | 2.807354922057604  | 0.0
+
+          idcg = 14.595390756454922 (sum of last column)
          */
         dcg = new DiscountedCumulativeGain(true, null, 10);
         assertEquals(EXPECTED_NDCG, dcg.evaluate("id", hits, rated).metricScore(), DELTA);
@@ -113,7 +113,7 @@ public void testDCGAt() {
     /**
      * This tests metric when some documents in the search result don't have a
      * rating provided by the user.
-     *
+     * 
          
      * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
      * -------------------------------------------------------------------------------------------
      * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 
@@ -122,7 +122,7 @@ public void testDCGAt() {
      * 4 | n/a | n/a | n/a | n/a
      * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163
      * 6 | n/a | n/a | n/a | n/a
-     *
+     * 
          
      * dcg = 12.779642067948913 (sum of last column)
      */
     public void testDCGAtSixMissingRatings() {
@@ -143,20 +143,20 @@ public void testDCGAtSixMissingRatings() {
         assertEquals(12.779642067948913, result.metricScore(), DELTA);
         assertEquals(2, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
-        /**
-         * Check with normalization: to get the maximal possible dcg, sort documents by
-         * relevance in descending order
-         *
-         * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
-         * ----------------------------------------------------------------------------------------
-         * 1 | 3 | 7.0 | 1.0  | 7.0
-         * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
-         * 3 | 2 | 3.0 | 2.0  | 1.5
-         * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339
-         * 5 | n.a | n.a | n.a.  | n.a.
-         * 6 | n.a | n.a | n.a  | n.a
-         *
-         * idcg = 13.347184833073591 (sum of last column)
+        /*
+          Check with normalization: to get the maximal possible dcg, sort documents by
+          relevance in descending order
+
+          rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
+          ----------------------------------------------------------------------------------------
+          1 | 3 | 7.0 | 1.0  | 7.0
+          2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
+          3 | 2 | 3.0 | 2.0  | 1.5
+          4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339
+          5 | n.a | n.a | n.a.  | n.a.
+          6 | n.a | n.a | n.a  | n.a
+
+          idcg = 13.347184833073591 (sum of last column)
          */
         dcg = new DiscountedCumulativeGain(true, null, 10);
         assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).metricScore(), DELTA);
@@ -166,7 +166,7 @@ public void testDCGAtSixMissingRatings() {
      * This tests that normalization works as expected when there are more rated
      * documents than search hits because we restrict DCG to be calculated at the
      * fourth position
-     *
+     * 
          
      * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
      * -------------------------------------------------------------------------------------------
      * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 
@@ -176,7 +176,7 @@ public void testDCGAtSixMissingRatings() {
      * -----------------------------------------------------------------
      * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163
      * 6 | n/a | n/a | n/a | n/a
-     *
+     * 
          
      * dcg = 12.392789260714371 (sum of last column until position 4)
      */
     public void testDCGAtFourMoreRatings() {
@@ -200,21 +200,21 @@ public void testDCGAtFourMoreRatings() {
         assertEquals(12.392789260714371, result.metricScore(), DELTA);
         assertEquals(1, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
-        /**
-         * Check with normalization: to get the maximal possible dcg, sort documents by
-         * relevance in descending order
-         *
-         * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
-         * ---------------------------------------------------------------------------------------
-         * 1 | 3 | 7.0 | 1.0  | 7.0
-         * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
-         * 3 | 2 | 3.0 | 2.0  | 1.5
-         * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339
-         * ---------------------------------------------------------------------------------------
-         * 5 | n.a | n.a | n.a.  | n.a.
-         * 6 | n.a | n.a | n.a  | n.a
-         *
-         * idcg = 13.347184833073591 (sum of last column)
+        /*
+          Check with normalization: to get the maximal possible dcg, sort documents by
+          relevance in descending order
+
+          rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1)
+          ---------------------------------------------------------------------------------------
+          1 | 3 | 7.0 | 1.0  | 7.0
+          2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
+          3 | 2 | 3.0 | 2.0  | 1.5
+          4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339
+          ---------------------------------------------------------------------------------------
+          5 | n.a | n.a | n.a.  | n.a.
+          6 | n.a | n.a | n.a  | n.a
+
+          idcg = 13.347184833073591 (sum of last column)
          */
         dcg = new DiscountedCumulativeGain(true, null, 10);
         assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).metricScore(), DELTA);
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index e2442e1f483f0..6ed486fbdb33b 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -206,7 +206,7 @@ public abstract class AbstractAsyncBulkByScrollAction<
 
     /**
      * Build the {@link BiFunction} to apply to all {@link RequestWrapper}.
-     *
+     * 
          
      * Public for testings....
      */
     public BiFunction, ScrollableHitSource.Hit, RequestWrapper> buildScriptApplier() {
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java
index d5a6e392f2019..7534de1408bcc 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/BulkByScrollParallelizationHelper.java
@@ -63,14 +63,14 @@ private BulkByScrollParallelizationHelper() {}
 
     /**
      * Takes an action created by a {@link BulkByScrollTask} and runs it with regard to whether the request is sliced or not.
-     *
+     * 
          
      * If the request is not sliced (i.e. the number of slices is 1), the worker action in the given {@link Runnable} will be started on
      * the local node. If the request is sliced (i.e. the number of slices is more than 1), then a subrequest will be created for each
      * slice and sent.
-     *
+     * 
          
      * If slices are set as {@code "auto"}, this class will resolve that to a specific number based on characteristics of the source
      * indices. A request with {@code "auto"} slices may end up being sliced or unsliced.
-     *
+     * 
          
      * This method is equivalent to calling {@link #initTaskState} followed by {@link #executeSlicedAction}
      */
     static > void startSlicedAction(
@@ -98,11 +98,11 @@ public void onFailure(Exception e) {
     /**
      * Takes an action and a {@link BulkByScrollTask} and runs it with regard to whether this task is a
      * leader or worker.
-     *
+     * 
          
      * If this task is a worker, the worker action in the given {@link Runnable} will be started on the local
      * node. If the task is a leader (i.e. the number of slices is more than 1), then a subrequest will be
      * created for each slice and sent.
-     *
+     * 
          
      * This method can only be called after the task state is initialized {@link #initTaskState}.
      */
     static > void executeSlicedAction(
@@ -125,7 +125,7 @@ static > void executeSliced
 
     /**
      * Takes a {@link BulkByScrollTask} and ensures that its initial task state (leader or worker) is set.
-     *
+     * 
          
      * If slices are set as {@code "auto"}, this method will resolve that to a specific number based on
      * characteristics of the source indices. A request with {@code "auto"} slices may end up being sliced or
      * unsliced. This method does not execute the action. In order to execute the action see
diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
index 6c8ca665424a6..826d4a7e5d61e 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
@@ -57,7 +57,7 @@
 
 /**
  * This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass.
- *
+ * 
          
  * As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing
  * a single node "cluster".
  */
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java
index 7d937157c1034..3c96affb7adf7 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpRequest.java
@@ -258,7 +258,7 @@ public FullHttpRequest nettyRequest() {
     /**
      * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications
      * and due to the underlying implementation, it performs case insensitive lookups of key to values.
-     *
+     * 
          
      * It is important to note that this implementation does have some downsides in that each invocation of the
      * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a
      * view of the underlying values.
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java
index b73f74d187a2a..4bab91565d3ad 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java
@@ -65,7 +65,7 @@
  * This class is adapted from {@link NioSocketChannel} class in the Netty project. It overrides the channel
  * read/write behavior to ensure that the bytes are always copied to a thread-local direct bytes buffer. This
  * happens BEFORE the call to the Java {@link SocketChannel} is issued.
- *
+ * 
          
  * The purpose of this class is to allow the disabling of netty direct buffer pooling while allowing us to
  * control how bytes end up being copied to direct memory. If we simply disabled netty pooling, we would rely
  * on the JDK's internal thread local buffer pooling. Instead, this class allows us to create a one thread
diff --git a/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/HaasePhonetik.java b/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/HaasePhonetik.java
index c80d32228feeb..b875fab7d4006 100644
--- a/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/HaasePhonetik.java
+++ b/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/HaasePhonetik.java
@@ -35,13 +35,13 @@
 /**
  * Geänderter Algorithmus aus der Matching Toolbox von Rainer Schnell
  * Java-Programmierung von Jörg Reiher
- *
+ * 
          
  * Die Kölner Phonetik wurde für den Einsatz in Namensdatenbanken wie
  * der Verwaltung eines Krankenhauses durch Martin Haase (Institut für
  * Sprachwissenschaft, Universität zu Köln) und Kai Heitmann (Insitut für
  * medizinische Statistik, Informatik und Epidemiologie, Köln)  überarbeitet.
  * M. Haase und K. Heitmann. Die Erweiterte Kölner Phonetik. 526, 2000.
- *
+ * 
          
  * nach: Martin Wilz, Aspekte der Kodierung phonetischer Ähnlichkeiten
  * in deutschen Eigennamen, Magisterarbeit.
  * http://www.uni-koeln.de/phil-fak/phonetik/Lehre/MA-Arbeiten/magister_wilz.pdf
diff --git a/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/KoelnerPhonetik.java b/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/KoelnerPhonetik.java
index 33e386af9f364..4bcc10ff73b0a 100644
--- a/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/KoelnerPhonetik.java
+++ b/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/KoelnerPhonetik.java
@@ -46,13 +46,13 @@
 
 /**
  * Kölner Phonetik
- *
+ * 
          
  * H.J. Postel, Die Kölner Phonetik. Ein Verfahren zu Identifizierung
  * von Personennamen auf der Grundlage der Gestaltanalyse. IBM-Nachrichten 19 (1969), 925-931
- *
+ * 
          
  * Algorithmus aus der Matching Toolbox von Rainer Schnell
  * Java-Programmierung von Jörg Reiher
- *
+ * 
          
  * mit Änderungen von Jörg Prante
  *
  */
diff --git a/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/Nysiis.java b/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/Nysiis.java
index 818dbba85e2de..c3237114c65d5 100644
--- a/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/Nysiis.java
+++ b/plugins/analysis-phonetic/src/main/java/org/opensearch/index/analysis/phonetic/Nysiis.java
@@ -40,7 +40,7 @@
 /**
  *
  * Taken from commons-codec trunk (unreleased yet)
- *
+ * 
          
  * Encodes a string into a NYSIIS value. NYSIIS is an encoding used to relate
  * similar names, but can also be used as a general purpose scheme to find word
  * with similar phonemes.
diff --git a/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java
index b4af9773f33de..2208c78bef67a 100644
--- a/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/opensearch/discovery/gce/GceDiscoveryTests.java
@@ -66,7 +66,6 @@
  * compute/v1/projects/[project-id]/zones/[zone]
  *
  * By default, project-id is the test method name, lowercase and missing the "test" prefix.
- *
  * For example, if you create a test `myNewAwesomeTest` with following settings:
  *
  * Settings nodeSettings = Settings.builder()
@@ -75,7 +74,6 @@
  *  .build();
  *
  *  You need to create a file under `src/test/resources/org/opensearch/discovery/gce/` named:
- *
  *  compute/v1/projects/mynewawesometest/zones/europe-west1-b/instances.json
  *
  */
diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/BasePerFieldCorrelationVectorsFormat.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/BasePerFieldCorrelationVectorsFormat.java
index 7763b1e42d63e..00b55eb75995c 100644
--- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/BasePerFieldCorrelationVectorsFormat.java
+++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/BasePerFieldCorrelationVectorsFormat.java
@@ -27,12 +27,12 @@
 public abstract class BasePerFieldCorrelationVectorsFormat extends PerFieldKnnVectorsFormat {
     /**
      * the hyper-parameters for constructing HNSW graphs.
-     * https://lucene.apache.org/core/9_4_0/core/org/apache/lucene/util/hnsw/HnswGraph.html
+     * HnswGraph.html
      */
     public static final String METHOD_PARAMETER_M = "m";
     /**
      * the hyper-parameters for constructing HNSW graphs.
-     * https://lucene.apache.org/core/9_4_0/core/org/apache/lucene/util/hnsw/HnswGraph.html
+     * HnswGraph.html
      */
     public static final String METHOD_PARAMETER_EF_CONSTRUCTION = "ef_construction";
 
diff --git a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java
index 134c6519d7220..3e567d0c04e53 100644
--- a/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java
+++ b/plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java
@@ -203,6 +203,7 @@ public void testDoToQueryInvalidFieldType() {
 
     /**
      * test serialization of Correlation Query Builder
+     * @throws Exception throws an IOException if serialization fails
      * @throws Exception Exception
      */
     public void testSerialization() throws Exception {
diff --git a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java
index 9255d3a2f299e..c8e9d8cfc0a12 100644
--- a/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java
+++ b/plugins/examples/custom-suggester/src/main/java/org/opensearch/example/customsuggester/CustomSuggestion.java
@@ -53,7 +53,6 @@ public class CustomSuggestion extends Suggest.Suggestion
 
     /**
      * An integer representing the type of the suggestion formerly used for internal serialization over the network.
-     *
      * This class is now serialized as a NamedWriteable and this value only remains for backwards compatibility
      */
     public static final int TYPE = 999;
@@ -106,7 +105,7 @@ public int getWriteableType() {
 
     /**
      * A meaningless value used to test that plugin suggesters can add fields to their Suggestion types
-     *
+     * 
          
      * This can't be serialized to xcontent because Suggestions appear in xcontent as an array of entries, so there is no place
      * to add a custom field. But we can still use a custom field internally and use it to define a Suggestion's behavior
      *
diff --git a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedClass.java b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedClass.java
index ca58d97377478..ee04741eb5a03 100644
--- a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedClass.java
+++ b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedClass.java
@@ -34,7 +34,7 @@
 
 /**
  * An example of a class to be allowlisted for use by painless scripts
- *
+ * 
          
  * Each of the members and methods below are allowlisted for use in search scripts.
  * See example_allowlist.txt.
  */
diff --git a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedInstance.java b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedInstance.java
index 0833f1f0c6659..35385e98187ea 100644
--- a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedInstance.java
+++ b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleAllowlistedInstance.java
@@ -34,7 +34,7 @@
 
 /**
  * An example of an instance to be allowlisted for use by painless scripts.
- *
+ * 
          
  * Each of the members and methods below are allowlisted for use in search scripts but only from this instance.
  */
 public class ExampleAllowlistedInstance {
diff --git a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExamplePainlessAnnotation.java b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExamplePainlessAnnotation.java
index 428a0cc688ad3..ac42c2b1cf9bc 100644
--- a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExamplePainlessAnnotation.java
+++ b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExamplePainlessAnnotation.java
@@ -34,7 +34,7 @@
 
 /**
  * An example of an annotation to be allowlisted for use by painless scripts
- *
+ * 
          
  * The annotation below is allowlisted for use in search scripts.
  * See example_allowlist.txt.
  */
diff --git a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleStaticMethodClass.java b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleStaticMethodClass.java
index 25443977a0f66..2f6d1ad9349bc 100644
--- a/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleStaticMethodClass.java
+++ b/plugins/examples/painless-allowlist/src/main/java/org/opensearch/example/painlessallowlist/ExampleStaticMethodClass.java
@@ -34,7 +34,7 @@
 
 /**
  * An example of a class with static methods to be allowlisted for use by painless scripts
- *
+ * 
          
  * The method below is allowlisted for use in search scripts.
  * See example_allowlist.txt.
  */
diff --git a/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
index 935887261dcc9..952cff96860f2 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -73,9 +73,9 @@
 import java.util.regex.Pattern;
 
 /** A {@link FieldMapper} for full-text fields with annotation markup e.g.
- *
+ * 
          
  *    "New mayor is [John Smith](type=person&value=John%20Smith) "
- *
+ * 
          
  * A special Analyzer wraps the default choice of analyzer in order
  * to strip the text field of annotation markup and inject the related
  * entity annotation tokens as supplementary tokens at the relevant points
diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java
index 82364d1b7b3c1..986720ec431fe 100644
--- a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java
+++ b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java
@@ -154,7 +154,7 @@ private static class AzureBlobStoreHttpHandler extends AzureHttpHandler implemen
 
     /**
      * HTTP handler that injects random Azure service errors
-     *
+     * 
          
      * Note: it is not a good idea to allow this handler to simulate too many errors as it would
      * slow down the test suite.
      */
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
index 9dcc312f8f5a7..b60701ba5e533 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
@@ -212,8 +212,8 @@ protected PasswordAuthentication getPasswordAuthentication() {
 
     /**
      * The location mode is not there in v12 APIs anymore but it is possible to mimic its semantics using
-     * retry options and combination of primary / secondary endpoints. Refer to migration guide for mode details:
-     * https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/storage/azure-storage-blob/migrationGuides/V8_V12.md#miscellaneous
+     * retry options and combination of primary / secondary endpoints. Refer to
+     * migration guide for mode details:
      */
     private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilder builder, final AzureStorageSettings settings) {
         final StorageConnectionString storageConnectionString = StorageConnectionString.create(settings.getConnectString(), logger);
@@ -335,8 +335,8 @@ private void closeInternally(ClientState state) {
     }
 
     /**
-     * Implements HTTP pipeline policy to collect statistics on API calls. See please:
-     * https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/storage/azure-storage-blob/migrationGuides/V8_V12.md#miscellaneous
+     * Implements HTTP pipeline policy to collect statistics on API calls. See :
+     * migration guide
      */
     private static class HttpStatsPolicy implements HttpPipelinePolicy {
         private final BiConsumer statsCollector;
diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
index 264888bb7da3a..bb0eafc7d1d4a 100644
--- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
@@ -451,7 +451,8 @@ private static RequestRetryPolicy requestRetryOptions(BlobServiceClient client)
     }
 
     /**
-     * Extract the blob name from a URI like https://myservice.azure.net/container/path/to/myfile
+     * Extract the blob name from a URI like :
+     * {@code https://myservice.azure.net/container/path/to/myfile }
      * It should remove the container part (first part of the path) and gives path/to/myfile
      * @param uri URI to parse
      * @return The blob name relative to the container
diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index f8e1a1cc39ae0..d223f7989c688 100644
--- a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -298,7 +298,7 @@ private static class GoogleCloudStorageBlobStoreHttpHandler extends GoogleCloudS
 
     /**
      * HTTP handler that injects random  Google Cloud Storage service errors
-     *
+     * 
          
      * Note: it is not a good idea to allow this handler to simulate too many errors as it would
      * slow down the test suite.
      */
diff --git a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsSecurityContext.java b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsSecurityContext.java
index 068adc2cc9991..07d1d29eecfc4 100644
--- a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsSecurityContext.java
+++ b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsSecurityContext.java
@@ -53,7 +53,7 @@
 
 /**
  * Oversees all the security specific logic for the HDFS Repository plugin.
- *
+ * 
          
  * Keeps track of the current user for a given repository, as well as which
  * permissions to grant the blob store restricted execution methods.
  */
diff --git a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
index 2758bd020e979..856cdf1eb565e 100644
--- a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
+++ b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
@@ -42,7 +42,7 @@
  * thread leaks out of the client and is picked up by the test framework. This thread filter is meant
  * to ignore the offending thread until a version of Hadoop is released that addresses the incorrect
  * interrupt handling.
- *
+ * 
          
  * In Hadoop 3.3.6, the org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap uses ForkJoinPool
  * to perform statistics calculation, leaving dangling workers.
  *
diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index 4e8e7720572e8..5f88ad7867513 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -225,7 +225,7 @@ private void validateAuthHeader(HttpExchange exchange) {
 
     /**
      * HTTP handler that injects random S3 service errors
-     *
+     * 
          
      * Note: it is not a good idea to allow this handler to simulate too many errors as it would
      * slow down the test suite.
      */
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java
index 933136228b1bb..2392c66329e06 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java
@@ -446,7 +446,7 @@ S3ClientSettings refine(Settings repositorySettings) {
 
     /**
      * Load all client settings from the given settings.
-     *
+     * 
          
      * Note this will always at least return a client named "default".
      */
     static Map load(final Settings settings, final Path configPath) {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java
index 3a35f6135f28b..d7e47e0ab1bcc 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java
@@ -54,7 +54,7 @@
  * Wrapper around an S3 object that will retry the {@link GetObjectRequest} if the download fails part-way through, resuming from where
  * the failure occurred. This should be handled by the SDK but it isn't today. This should be revisited in the future (e.g. before removing
  * the {@code LegacyESVersion#V_7_0_0} version constant) and removed when the SDK handles retries itself.
- *
+ * 
          
  * See https://github.com/aws/aws-sdk-java/issues/856 for the related SDK issue
  */
 class S3RetryingInputStream extends InputStream {
diff --git a/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbMMapDirectoryTests.java b/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbMMapDirectoryTests.java
index 2cac58262c75a..e1655cc5e0784 100644
--- a/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbMMapDirectoryTests.java
+++ b/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbMMapDirectoryTests.java
@@ -47,9 +47,9 @@ protected Directory getDirectory(Path file) throws IOException {
 
     @Override
     public void testCreateOutputForExistingFile() throws IOException {
-        /**
-         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
-         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+        /*
+          This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+          with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
          */
     }
 }
diff --git a/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbNIOFSDirectoryTests.java b/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbNIOFSDirectoryTests.java
index 7390759029dfc..6f821147c3079 100644
--- a/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbNIOFSDirectoryTests.java
+++ b/plugins/store-smb/src/test/java/org/opensearch/index/store/SmbNIOFSDirectoryTests.java
@@ -26,9 +26,9 @@ protected Directory getDirectory(Path file) throws IOException {
 
     @Override
     public void testCreateOutputForExistingFile() throws IOException {
-        /**
-         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
-         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+        /*
+          This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+          with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
          */
     }
 }
diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java
index d25ef33c2ce29..5abd6f2710198 100644
--- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java
+++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpRequest.java
@@ -257,7 +257,7 @@ public FullHttpRequest nettyRequest() {
     /**
      * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications
      * and due to the underlying implementation, it performs case insensitive lookups of key to values.
-     *
+     * 
          
      * It is important to note that this implementation does have some downsides in that each invocation of the
      * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a
      * view of the underlying values.
diff --git a/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java
index aabc3aee8887f..2675e9b62de35 100644
--- a/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/QueryBuilderBWCIT.java
@@ -78,7 +78,7 @@
  * PercolatorFieldMapper#createQueryBuilderField(...) method). Using the query builders writable contract. This test
  * does best effort verifying that we don't break bwc for query builders between the first previous major version and
  * the latest current major release.
- *
+ * 
          
  * The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the
  * json format of a query being tested here then feel free to change this.
  */
diff --git a/qa/no-bootstrap-tests/src/test/java/org/opensearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/opensearch/bootstrap/SpawnerNoBootstrapTests.java
index c3c332aecfd4c..8ca90791f649e 100644
--- a/qa/no-bootstrap-tests/src/test/java/org/opensearch/bootstrap/SpawnerNoBootstrapTests.java
+++ b/qa/no-bootstrap-tests/src/test/java/org/opensearch/bootstrap/SpawnerNoBootstrapTests.java
@@ -65,7 +65,7 @@
 
 /**
  * Create a simple "daemon controller", put it in the right place and check that it runs.
- *
+ * 
          
  * Extends LuceneTestCase rather than OpenSearchTestCase as OpenSearchTestCase installs a system call filter, and
  * that prevents the Spawner class from doing its job. Also needs to run in a separate JVM to other
  * tests that extend OpenSearchTestCase for the same reason.
diff --git a/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java
index 02a613be320c2..4bb3877fc04a8 100644
--- a/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java
+++ b/qa/os/src/test/java/org/opensearch/packaging/test/PackagingTestCase.java
@@ -441,7 +441,7 @@ public static Path createTempDir(String prefix) throws IOException {
 
     /**
      * Run the given action with a temporary copy of the config directory.
-     *
+     * 
          
      * Files under the path passed to the action may be modified as necessary for the
      * test to execute, and running OpenSearch with {@link #startOpenSearch()} will
      * use the temporary directory.
diff --git a/qa/os/src/test/java/org/opensearch/packaging/util/FileMatcher.java b/qa/os/src/test/java/org/opensearch/packaging/util/FileMatcher.java
index 7904d1a046916..958de24848178 100644
--- a/qa/os/src/test/java/org/opensearch/packaging/util/FileMatcher.java
+++ b/qa/os/src/test/java/org/opensearch/packaging/util/FileMatcher.java
@@ -51,7 +51,7 @@
 /**
  * Asserts that a file at a path matches its status as Directory/File, and its owner. If on a posix system, also matches the permission
  * set is what we expect.
- *
+ * 
          
  * This class saves information about its failed matches in instance variables and so instances should not be reused
  */
 public class FileMatcher extends TypeSafeMatcher {
diff --git a/qa/os/src/test/java/org/opensearch/packaging/util/Installation.java b/qa/os/src/test/java/org/opensearch/packaging/util/Installation.java
index 25cefa948ff10..26af39d66cad3 100644
--- a/qa/os/src/test/java/org/opensearch/packaging/util/Installation.java
+++ b/qa/os/src/test/java/org/opensearch/packaging/util/Installation.java
@@ -137,7 +137,7 @@ public static Installation ofContainer(Shell sh, Distribution distribution) {
 
     /**
      * Returns the user that owns this installation.
-     *
+     * 
          
      * For packages this is root, and for archives it is the user doing the installation.
      */
     public String getOwner() {
diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java
index aef363058b394..f963f8d221bb5 100644
--- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java
@@ -98,11 +98,11 @@ private void waitForSearchableDocs(String index, int shardCount, int replicaCoun
 
         // Verify segment store
         assertBusy(() -> {
-            /**
-             * Use default tabular output and sort response based on shard,segment,primaryOrReplica columns to allow line by
-             * line parsing where records related to a segment (e.g. _0) are chunked together with first record belonging
-             * to primary while remaining *replicaCount* records belongs to replica copies
-             * */
+            /*
+              Use default tabular output and sort response based on shard,segment,primaryOrReplica columns to allow line by
+              line parsing where records related to a segment (e.g. _0) are chunked together with first record belonging
+              to primary while remaining *replicaCount* records belongs to replica copies
+              */
             Request segrepStatsRequest = new Request("GET", "/_cat/segments/" + index + "?s=shard,segment,primaryOrReplica");
             segrepStatsRequest.addParameter("h", "index,shard,primaryOrReplica,segment,docs.count");
             Response segrepStatsResponse = client().performRequest(segrepStatsRequest);
@@ -259,7 +259,8 @@ public void testIndexing() throws Exception {
      * This test verifies that during rolling upgrades the segment replication does not break when replica shards can
      * be running on older codec versions.
      *
-     * @throws Exception exception
+     * @throws Exception if index creation fail
+     * @throws UnsupportedOperationException if cluster type is unknown
      */
     @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7679")
     public void testIndexingWithSegRep() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/HotThreadsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/HotThreadsIT.java
index 6343bd127c458..4c9f49df71257 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/HotThreadsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/HotThreadsIT.java
@@ -60,8 +60,8 @@
 public class HotThreadsIT extends OpenSearchIntegTestCase {
 
     public void testHotThreadsDontFail() throws ExecutionException, InterruptedException {
-        /**
-         * This test just checks if nothing crashes or gets stuck etc.
+        /*
+          This test just checks if nothing crashes or gets stuck etc.
          */
         createIndex("test");
         final int iters = scaledRandomIntBetween(2, 20);
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/AbstractTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/AbstractTasksIT.java
index 0197ccf059737..44ba585016d8e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/AbstractTasksIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/AbstractTasksIT.java
@@ -112,7 +112,7 @@ protected int numberOfEvents(String actionMasks, Function findEvents(String actionMasks, Function, Boolean> criteria) {
         List events = new ArrayList<>();
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java
index c733329a1b5f7..e6fd9139d45f2 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java
@@ -30,7 +30,7 @@
 
 /**
  * Integration tests for task management API with Concurrent Segment Search
- *
+ * 
          
  * The way the test framework bootstraps the test cluster makes it difficult to parameterize the feature flag.
  * Once concurrent search is moved behind a cluster setting we can parameterize these tests behind the setting.
  */
@@ -72,7 +72,7 @@ protected Settings featureFlagSettings() {
 
     /**
      * Tests the number of threads that worked on a search task.
-     *
+     * 
          
      * Currently, we try to control concurrency by creating an index with 7 segments and rely on
      * the way concurrent search creates leaf slices from segments. Once more concurrency controls are introduced
      * we should improve this test to use those methods.
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java
index aff7c5d9876ac..36fe3748e9d10 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java
@@ -46,7 +46,7 @@
 
 /**
  * This class tests that repository operations (Put, Delete, Verify) are blocked when the cluster is read-only.
- *
+ * 
          
  * The @NodeScope TEST is needed because this class updates the cluster setting "cluster.blocks.read_only".
  */
 @ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST)
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
index 347011721c728..78fb01b07b6b1 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
@@ -53,7 +53,7 @@
 
 /**
  * This class tests that snapshot operations (Create, Delete, Restore) are blocked when the cluster is read-only.
- *
+ * 
          
  * The @NodeScope TEST is needed because this class updates the cluster setting "cluster.blocks.read_only".
  */
 @ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST)
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java
index 737c0acc309fd..cd6cb0ca3b172 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorRetryIT.java
@@ -194,7 +194,7 @@ private static void indexDocs(BulkProcessor processor, int numDocs) {
     /**
      * Internal helper class to correlate backoff states with bulk responses. This is needed to check whether we maxed out the number
      * of retries but still got rejected (which is perfectly fine and can also happen from time to time under heavy load).
-     *
+     * 
          
      * This implementation relies on an implementation detail in Retry, namely that the bulk listener is notified on the same thread
      * as the last call to the backoff policy's iterator. The advantage is that this is non-invasive to the rest of the production code.
      */
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java b/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java
index c62c61d5919d6..aefabcb9bc14f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/ingest/AsyncIngestProcessorIT.java
@@ -69,7 +69,7 @@
 /**
  * The purpose of this test is to verify that when a processor executes an operation asynchronously that
  * the expected result is the same as if the same operation happens synchronously.
- *
+ * 
          
  * In this test two test processor are defined that basically do the same operation, but a single processor
  * executes asynchronously. The result of the operation should be the same and also the order in which the
  * bulk responses are returned should be the same as how the corresponding index requests were defined.
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java
index f0a3b5a5901ce..b1934f901ac65 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java
@@ -109,8 +109,8 @@ public List getAggregations() {
 
         @Override
         public List getFetchSubPhases(FetchPhaseConstructionContext context) {
-            /**
-             * Set up a fetch sub phase that throws an exception on indices whose name that start with "boom".
+            /*
+              Set up a fetch sub phase that throws an exception on indices whose name that start with "boom".
              */
             return Collections.singletonList(fetchContext -> new FetchSubPhaseProcessor() {
                 @Override
diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterDisruptionIT.java
index 38b86d307d197..737b272613a44 100644
--- a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterDisruptionIT.java
@@ -461,7 +461,7 @@ public boolean validateClusterForming() {
 
     /**
      * Tests that indices are properly deleted even if there is a cluster-manager transition in between.
-     * Test for https://github.com/elastic/elasticsearch/issues/11665
+     * Test for Elasticsearch issue #11665
      */
     public void testIndicesDeleted() throws Exception {
         final String idxName = "test";
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java
index bdefd7a5e199a..f485d4e402b41 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java
@@ -91,7 +91,7 @@ public void testGlobalPrimaryAllocation() throws Exception {
 
     /**
      * This test verifies the happy path where primary shard allocation is balanced when multiple indices are created.
-     *
+     * 
          
      * This test in general passes without primary shard balance as well due to nature of allocation algorithm which
      * assigns all primary shards first followed by replica copies.
      */
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java
index ae88dd76d54e0..547f9e7a8d380 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java
@@ -509,7 +509,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
     }
 
     /**
-     * Test for https://github.com/elastic/elasticsearch/issues/47276 which checks that the persisted metadata on a data node does not
+     * Test for Elasticsearch issue #47276 which checks that the persisted metadata on a data node does not
      * become inconsistent when using replicated closed indices.
      */
     public void testRelocatedClosedIndexIssue() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java
index f636185fd4649..d28df90216beb 100644
--- a/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/recovery/FullRollingRestartIT.java
@@ -198,11 +198,11 @@ public void testNoRebalanceOnRollingRestart() throws Exception {
         // see https://github.com/elastic/elasticsearch/issues/14387
         internalCluster().startClusterManagerOnlyNode(Settings.EMPTY);
         internalCluster().startDataOnlyNodes(3);
-        /**
-         * We start 3 nodes and a dedicated cluster-manager. Restart on of the data-nodes and ensure that we got no relocations.
-         * Yet we have 6 shards 0 replica so that means if the restarting node comes back both other nodes are subject
-         * to relocating to the restarting node since all had 2 shards and now one node has nothing allocated.
-         * We have a fix for this to wait until we have allocated unallocated shards now so this shouldn't happen.
+        /*
+          We start 3 nodes and a dedicated cluster-manager. Restart on of the data-nodes and ensure that we got no relocations.
+          Yet we have 6 shards 0 replica so that means if the restarting node comes back both other nodes are subject
+          to relocating to the restarting node since all had 2 shards and now one node has nothing allocated.
+          We have a fix for this to wait until we have allocated unallocated shards now so this shouldn't happen.
          */
         prepareCreate("test").setSettings(
             Settings.builder()
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java
index 895e24ba65132..ee94e574228df 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java
@@ -1481,7 +1481,7 @@ public void testExceptionOnNegativeInterval() {
     /**
      * https://github.com/elastic/elasticsearch/issues/31760 shows an edge case where an unmapped "date" field in two indices
      * that are queried simultaneously can lead to the "format" parameter in the aggregation not being preserved correctly.
-     *
+     * 
          
      * The error happens when the bucket from the "unmapped" index is received first in the reduce phase, however the case can
      * be recreated when aggregating about a single index with an unmapped date field and also getting "empty" buckets.
      */
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java
index bb90c1294ecb8..dc3b690c7f78f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java
@@ -544,13 +544,13 @@ public void testNested() throws Exception {
 
     /**
      * https://github.com/elastic/elasticsearch/issues/33514
-     *
+     * 
          
      * This bug manifests as the max_bucket agg ("peak") being added to the response twice, because
      * the pipeline agg is run twice.  This makes invalid JSON and breaks conversion to maps.
      * The bug was caused by an UnmappedTerms being the chosen as the first reduction target.  UnmappedTerms
      * delegated reduction to the first non-unmapped agg, which would reduce and run pipeline aggs.  But then
      * execution returns to the UnmappedTerms and _it_ runs pipelines as well, doubling up on the values.
-     *
+     * 
          
      * Applies to any pipeline agg, not just max.
      */
     public void testFieldIsntWrittenOutTwice() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java
index 0cf89778c6e99..2aad0d2d38901 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java
@@ -1168,7 +1168,7 @@ public void testHoltWintersMinimization() {
      * the default settings.  Which means our mock histo will match the generated result (which it won't
      * if the minimizer is actually working, since the coefficients will be different and thus generate different
      * data)
-     *
+     * 
          
      * We can simulate this by setting the window size == size of histo
      */
     public void testMinimizeNotEnoughData() {
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java
index 43b7179a335f8..4a178e7066846 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java
@@ -93,8 +93,8 @@ public void testDeletePit() throws Exception {
             assertTrue(deletePitInfo.isSuccessful());
         }
         validatePitStats("index", 0, 10);
-        /**
-         * Checking deleting the same PIT id again results in succeeded
+        /*
+          Checking deleting the same PIT id again results in succeeded
          */
         deleteExecute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
         deletePITResponse = deleteExecute.get();
@@ -113,8 +113,8 @@ public void testDeletePitWithValidAndDeletedIds() throws Exception {
         pitIds.add(pitResponse.getId());
         validatePitStats("index", 5, 0);
 
-        /**
-         * Delete Pit #1
+        /*
+          Delete Pit #1
          */
         DeletePitRequest deletePITRequest = new DeletePitRequest(pitIds);
         ActionFuture deleteExecute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
@@ -128,8 +128,8 @@ public void testDeletePitWithValidAndDeletedIds() throws Exception {
         pitResponse = execute.get();
         pitIds.add(pitResponse.getId());
         validatePitStats("index", 5, 5);
-        /**
-         * Delete PIT with both Ids #1 (which is deleted) and #2 (which is present)
+        /*
+          Delete PIT with both Ids #1 (which is deleted) and #2 (which is present)
          */
         deletePITRequest = new DeletePitRequest(pitIds);
         deleteExecute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
@@ -165,9 +165,9 @@ public void testDeleteAllPits() throws Exception {
         validatePitStats("index1", 5, 0);
         DeletePitRequest deletePITRequest = new DeletePitRequest("_all");
 
-        /**
-         * When we invoke delete again, returns success after clearing the remaining readers. Asserting reader context
-         * not found exceptions don't result in failures ( as deletion in one node is successful )
+        /*
+          When we invoke delete again, returns success after clearing the remaining readers. Asserting reader context
+          not found exceptions don't result in failures ( as deletion in one node is successful )
          */
         ActionFuture execute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
         DeletePitResponse deletePITResponse = execute.get();
@@ -207,9 +207,9 @@ public Settings onNodeStopped(String nodeName) throws Exception {
         });
 
         ensureGreen();
-        /**
-         * When we invoke delete again, returns success after clearing the remaining readers. Asserting reader context
-         * not found exceptions don't result in failures ( as deletion in one node is successful )
+        /*
+          When we invoke delete again, returns success after clearing the remaining readers. Asserting reader context
+          not found exceptions don't result in failures ( as deletion in one node is successful )
          */
         ActionFuture execute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
         DeletePitResponse deletePITResponse = execute.get();
@@ -242,9 +242,9 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             }
         });
         ensureGreen();
-        /**
-         * When we invoke delete again, returns success as all readers are cleared. (Delete all on node which is Up and
-         * once the node restarts, all active contexts are cleared in the node )
+        /*
+          When we invoke delete again, returns success as all readers are cleared. (Delete all on node which is Up and
+          once the node restarts, all active contexts are cleared in the node )
          */
         ActionFuture execute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
         DeletePitResponse deletePITResponse = execute.get();
@@ -278,8 +278,8 @@ public void testDeleteWhileSearch() throws Exception {
                         }
                     }
                 } catch (Exception e) {
-                    /**
-                     * assert for exception once delete pit goes through. throw error in case of any exeption before that.
+                    /*
+                      assert for exception once delete pit goes through. throw error in case of any exeption before that.
                      */
                     if (deleted.get() == true) {
                         Throwable t = ExceptionsHelper.unwrapCause(e.getCause());
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java
index 00ac574b8bd72..b99f66850e9e3 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java
@@ -220,8 +220,8 @@ public void testPitWithSearchAfter() throws Exception {
             .setPointInTime(new PointInTimeBuilder(pitResponse.getId()))
             .get();
         assertEquals(3, sr.getHits().getHits().length);
-        /**
-         * Add new data and assert PIT results remain the same and normal search results gets refreshed
+        /*
+          Add new data and assert PIT results remain the same and normal search results gets refreshed
          */
         indexRandom(true, client().prepareIndex("test").setId("4").setSource("field1", 102));
         sr = client().prepareSearch()
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java
index 7880fc24fd846..1b8bd9694483d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -91,16 +91,16 @@ protected boolean forbidPrivateIndexSettings() {
     }
 
     public void testManyToManyGeoPoints() throws ExecutionException, InterruptedException, IOException {
-        /**
-         * | q  |  d1    |   d2
-         * |    |        |
-         * |    |        |
-         * |    |        |
-         * |2  o|  x     |     x
-         * |    |        |
-         * |1  o|      x | x
-         * |___________________________
-         * 1   2   3   4   5   6   7
+        /*
+          | q  |  d1    |   d2
+          |    |        |
+          |    |        |
+          |    |        |
+          |2  o|  x     |     x
+          |    |        |
+          |1  o|      x | x
+          |___________________________
+          1   2   3   4   5   6   7
          */
         Version version = randomBoolean() ? Version.CURRENT : VersionUtils.randomIndexCompatibleVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
@@ -187,11 +187,10 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce
     }
 
     public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedException, IOException {
-        /**
-         * q  = (0, 0)
-         *
-         * d1 = (0, 1), (0, 4), (0, 10); so avg. distance is 5, median distance is 4
-         * d2 = (0, 1), (0, 5), (0, 6); so avg. distance is 4, median distance is 5
+        /*
+          q  = (0, 0)
+          d1 = (0, 1), (0, 4), (0, 10); so avg. distance is 5, median distance is 4
+          d2 = (0, 1), (0, 5), (0, 6); so avg. distance is 4, median distance is 5
          */
         Version version = randomBoolean() ? Version.CURRENT : VersionUtils.randomIndexCompatibleVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
@@ -256,16 +255,17 @@ protected void createShuffeldJSONArray(XContentBuilder builder, GeoPoint[] point
     }
 
     public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionException, InterruptedException, IOException {
-        /**   q     d1       d2
-         * |4  o|   x    |   x
-         * |    |        |
-         * |3  o|  x     |  x
-         * |    |        |
-         * |2  o| x      | x
-         * |    |        |
-         * |1  o|x       |x
-         * |______________________
-         * 1   2   3   4   5   6
+        /*
+           q     d1       d2
+          |4  o|   x    |   x
+          |    |        |
+          |3  o|  x     |  x
+          |    |        |
+          |2  o| x      | x
+          |    |        |
+          |1  o|x       |x
+          |______________________
+          1   2   3   4   5   6
          */
         Version version = randomBoolean() ? Version.CURRENT : VersionUtils.randomIndexCompatibleVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 5268d5ddf35af..91b0aa6438753 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -1884,7 +1884,7 @@ public void testCannotCreateSnapshotsWithSameName() throws Exception {
      * This test ensures that when a shard is removed from a node (perhaps due to the node
      * leaving the cluster, then returning), all snapshotting of that shard is aborted, so
      * all Store references held onto by the snapshot are released.
-     *
+     * 
          
      * See https://github.com/elastic/elasticsearch/issues/20876
      */
     public void testSnapshotCanceledOnRemovedShard() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java
index c651689e21d3d..7f016caf22149 100644
--- a/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/versioning/ConcurrentSeqNoVersioningIT.java
@@ -93,7 +93,7 @@
  *     provided the primaryTerm and seqNo still matches. The reason we cannot assume it will not take place after receiving the failure
  *     is that a request can fork into two because of retries on disconnect, and now race against itself. The retry might complete (and do a
  *     dirty or stale read) before the forked off request gets to execute, and that one might still subsequently succeed.
- *
+ * 
          
  *     Such writes are not necessarily fully replicated and can be lost. There is no
  *     guarantee that the previous value did not have the specified primaryTerm and seqNo
          • 
  *     
        • CAS writes with other exceptions might or might not have taken place. If they have taken place, then after invocation but not
diff --git a/server/src/main/java/org/apache/lucene/index/OneMergeHelper.java b/server/src/main/java/org/apache/lucene/index/OneMergeHelper.java
index 8f67bb87b5c42..0ada02a09d157 100644
--- a/server/src/main/java/org/apache/lucene/index/OneMergeHelper.java
+++ b/server/src/main/java/org/apache/lucene/index/OneMergeHelper.java
@@ -59,8 +59,8 @@ public static double getMbPerSec(Thread thread, MergePolicy.OneMerge merge) {
      * Returns total bytes written by this merge.
      **/
     public static long getTotalBytesWritten(Thread thread, MergePolicy.OneMerge merge) throws IOException {
-        /**
-         * TODO: The number of bytes written during the merge should be accessible in OneMerge.
+        /*
+          TODO: The number of bytes written during the merge should be accessible in OneMerge.
          */
         if (thread instanceof ConcurrentMergeScheduler.MergeThread) {
             return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter.getTotalBytesWritten();
diff --git a/server/src/main/java/org/apache/lucene/queryparser/classic/XQueryParser.java b/server/src/main/java/org/apache/lucene/queryparser/classic/XQueryParser.java
index 9a8c295d60ec7..2f00ea69207a7 100644
--- a/server/src/main/java/org/apache/lucene/queryparser/classic/XQueryParser.java
+++ b/server/src/main/java/org/apache/lucene/queryparser/classic/XQueryParser.java
@@ -38,7 +38,7 @@
 /**
  * This class is just a workaround to make {@link QueryParser#handleBareFuzzy(String, Token, String)} accessible by sub-classes.
  * It is needed for {@link QueryParser}s that need to override the parsing of the slop in a fuzzy query (e.g. word~2, word~).
- *
+ * 
          
  * TODO: We should maybe rewrite this with the flexible query parser which matches the same syntax with more freedom.
  */
 public class XQueryParser extends QueryParser {
diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java
index e4c299ba572b1..9ca0491bc29f5 100644
--- a/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java
+++ b/server/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java
@@ -50,7 +50,7 @@
  * A collector that groups documents based on field values and returns {@link CollapseTopFieldDocs}
  * output. The collapsing is done in a single pass by selecting only the top sorted document per collapse key.
  * The value used for the collapse key of each group can be found in {@link CollapseTopFieldDocs#collapseValues}.
- *
+ * 
          
  * TODO: If the sort is based on score we should propagate the mininum competitive score when orderedGroups is full.
  * This is safe for collapsing since the group sort is the same as the query sort.
  */
diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java b/server/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java
index 6fde39b16a59a..4edcdea42b53b 100644
--- a/server/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java
+++ b/server/src/main/java/org/apache/lucene/search/uhighlight/BoundedBreakIteratorScanner.java
@@ -43,13 +43,13 @@
  * If the {@link BreakIterator} cannot find a passage smaller than the maximum length,
  * a secondary break iterator is used to re-split the passage at the first boundary after
  * maximum length.
- *
+ * 
          
  * This is useful to split passages created by {@link BreakIterator}s like `sentence` that
  * can create big outliers on semi-structured text.
- *
+ * 
          
  *
  * WARNING: This break iterator is designed to work with the {@link UnifiedHighlighter}.
- *
+ * 
          
  * TODO: We should be able to create passages incrementally, starting from the offset of the first match and expanding or not
  * depending on the offsets of subsequent matches. This is currently impossible because {@link FieldHighlighter} uses
  * only the first matching offset to derive the start and end of each passage.
diff --git a/server/src/main/java/org/apache/lucene/util/packed/XPacked64.java b/server/src/main/java/org/apache/lucene/util/packed/XPacked64.java
index 4777b77cfbfed..9e9f6d1fd817d 100644
--- a/server/src/main/java/org/apache/lucene/util/packed/XPacked64.java
+++ b/server/src/main/java/org/apache/lucene/util/packed/XPacked64.java
@@ -38,8 +38,8 @@
 /**
  * Forked from Lucene 8.x; removed in Lucene 9.0
  *
- * @todo further investigate a better alternative
- *
+ * TODO: further investigate a better alternative
+ * 
          
  * Space optimized random access capable array of values with a fixed number of bits/value. Values
  * are packed contiguously.
  *
diff --git a/server/src/main/java/org/apache/lucene/util/packed/XPacked64SingleBlock.java b/server/src/main/java/org/apache/lucene/util/packed/XPacked64SingleBlock.java
index 0324522e9a68d..53cf4ed8e2273 100644
--- a/server/src/main/java/org/apache/lucene/util/packed/XPacked64SingleBlock.java
+++ b/server/src/main/java/org/apache/lucene/util/packed/XPacked64SingleBlock.java
@@ -25,8 +25,8 @@
 /**
  * Forked from Lucene 8.x; removed in Lucene 9.0
  *
- * @todo further investigate a better alternative
- *
+ * TODO: further investigate a better alternative
+ * 
          
  * This class is similar to {@link Packed64} except that it trades space for speed by ensuring that
  * a single block needs to be read/written in order to read/write a value.
  */
diff --git a/server/src/main/java/org/apache/lucene/util/packed/XPackedInts.java b/server/src/main/java/org/apache/lucene/util/packed/XPackedInts.java
index f94a4531a7db9..4260d34ead7c9 100644
--- a/server/src/main/java/org/apache/lucene/util/packed/XPackedInts.java
+++ b/server/src/main/java/org/apache/lucene/util/packed/XPackedInts.java
@@ -35,9 +35,9 @@
 
 /**
  * Forked from Lucene 8.x; removed in Lucene 8.9
- *
+ * 
          
  * Todo: further investigate a better alternative
- *
+ * 
          
  * Simplistic compression for array of unsigned long values. Each value is {@code >= 0} and {@code
  * <=} a specified maximum value. The values are stored as packed ints, with each value consuming a
  * fixed number of bits.
diff --git a/server/src/main/java/org/opensearch/action/AliasesRequest.java b/server/src/main/java/org/opensearch/action/AliasesRequest.java
index 4c5d5628b1aac..3632ba2d7304f 100644
--- a/server/src/main/java/org/opensearch/action/AliasesRequest.java
+++ b/server/src/main/java/org/opensearch/action/AliasesRequest.java
@@ -54,7 +54,7 @@ public interface AliasesRequest extends IndicesRequest.Replaceable {
 
     /**
      * Replaces current aliases with the provided aliases.
-     *
+     * 
          
      * Sometimes aliases expressions need to be resolved to concrete aliases prior to executing the transport action.
      */
     void replaceAliases(String... aliases);
diff --git a/server/src/main/java/org/opensearch/action/DocWriteRequest.java b/server/src/main/java/org/opensearch/action/DocWriteRequest.java
index df6414470ab6b..31f61e76c74ff 100644
--- a/server/src/main/java/org/opensearch/action/DocWriteRequest.java
+++ b/server/src/main/java/org/opensearch/action/DocWriteRequest.java
@@ -149,7 +149,7 @@ public interface DocWriteRequest extends IndicesRequest, Accountable {
 
     /**
      * If set, only perform this request if the document was last modification was assigned this primary term.
-     *
+     * 
          
      * If the document last modification was assigned a different term a
      * {@link org.opensearch.index.engine.VersionConflictEngineException} will be thrown.
      */
diff --git a/server/src/main/java/org/opensearch/action/DocWriteResponse.java b/server/src/main/java/org/opensearch/action/DocWriteResponse.java
index afdb1d3a0bdd9..e3052b3b80035 100644
--- a/server/src/main/java/org/opensearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/opensearch/action/DocWriteResponse.java
@@ -341,7 +341,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
 
     /**
      * Parse the output of the {@link #innerToXContent(XContentBuilder, Params)} method.
-     *
+     * 
          
      * This method is intended to be called by subclasses and must be called multiple times to parse all the information concerning
      * {@link DocWriteResponse} objects. It always parses the current token, updates the given parsing context accordingly
      * if needed and then immediately returns.
diff --git a/server/src/main/java/org/opensearch/action/TaskOperationFailure.java b/server/src/main/java/org/opensearch/action/TaskOperationFailure.java
index 0930bd2741810..5948dd3e2b7cb 100644
--- a/server/src/main/java/org/opensearch/action/TaskOperationFailure.java
+++ b/server/src/main/java/org/opensearch/action/TaskOperationFailure.java
@@ -50,7 +50,7 @@
 
 /**
  * Information about task operation failures
- *
+ * 
          
  * The class is final due to serialization limitations
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
index 625aa91e6ea7f..3dec781f0acf4 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
@@ -95,7 +95,7 @@ public ClusterAllocationExplainRequest(StreamInput in) throws IOException {
      * Create a new allocation explain request. If {@code primary} is false, the first unassigned replica
      * will be picked for explanation. If no replicas are unassigned, the first assigned replica will
      * be explained.
-     *
+     * 
          
      * Package private for testing.
      */
     ClusterAllocationExplainRequest(String index, int shard, boolean primary, @Nullable String currentNode) {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index 8d82827f4ee50..e62c83490d810 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -67,7 +67,7 @@
 
 /**
  * ActionType to get a single task. If the task isn't running then it'll try to request the status from request index.
- *
+ * 
          
  * The general flow is:
  * 
            
  * 
          • If this isn't being executed on the node to which the requested TaskId belongs then move to that node.
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java
index f26ccbc569cf0..774bffa10da4f 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java
@@ -67,7 +67,7 @@
 
 /**
  * Repository cleanup action for repository implementations based on {@link BlobStoreRepository}.
- *
+ * 
            
  * The steps taken by the repository cleanup operation are as follows:
  * 
              
  *     
            1. Check that there are no running repository cleanup, snapshot create, or snapshot delete actions
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java
index ac975e917e056..6fb086f65497e 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java
@@ -88,7 +88,7 @@ public ActionRequestValidationException validate() {
     /**
      * The names of the repositories.
      *
-     * @return list of repositories
+     * @return  array of repository names
      */
     public String[] repositories() {
         return this.repositories;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 9f8fc6067f405..0cfc24994768a 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -261,7 +261,7 @@ public CreateSnapshotRequest indices(List indices) {
     /**
      * Returns a list of indices that should be included into the snapshot
      *
-     * @return list of indices
+     * @return array of index names
      */
     @Override
     public String[] indices() {
@@ -316,7 +316,7 @@ public CreateSnapshotRequest partial(boolean partial) {
 
     /**
      * If set to true the operation should wait for the snapshot completion before returning.
-     *
+     * 
              
      * By default, the operation will return as soon as snapshot is initialized. It can be changed by setting this
      * flag to true.
      *
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 256850da0d503..c688b13ed9fdd 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -497,7 +497,7 @@ public Settings indexSettings() {
      * this is the snapshot that this request restores. If the client can only identify a snapshot by its name then there is a risk that the
      * desired snapshot may be deleted and replaced by a new snapshot with the same name which is inconsistent with the original one. This
      * method lets us fail the restore if the precise snapshot we want is not available.
-     *
+     * 
              
      * This is for internal use only and is not exposed in the REST layer.
      */
     public RestoreSnapshotRequest snapshotUuid(String snapshotUuid) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java
index 7cda041b7d9cf..814a65e2a5bf0 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java
@@ -81,7 +81,7 @@ public Builder addIndices(String... indices) {
 
     /**
      * Specifies what type of requested indices to ignore and wildcard indices expressions.
-     *
+     * 
              
      * For example indices that don't exist.
      */
     @SuppressWarnings("unchecked")
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
index 7cda0c1948d24..d048a3008f588 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
@@ -243,7 +243,7 @@ public CreateIndexRequest settings(Map source) {
 
     /**
      * Set the mapping for this index
-     *
+     * 
              
      * The mapping should be in the form of a JSON string, with an outer _doc key
      * 
                    *     .mapping("{\"_doc\":{\"properties\": ... }}")
@@ -269,7 +269,7 @@ public CreateIndexRequest mapping(String source, XContentType xContentType) {
 
     /**
      * Adds mapping that will be added when the index gets created.
-     *
+     * 
              
      * Note that the definition should *not* be nested under a type name.
      *
      * @param source The mapping source
@@ -296,7 +296,7 @@ private CreateIndexRequest mapping(BytesReference source, XContentType xContentT
 
     /**
      * Adds mapping that will be added when the index gets created.
-     *
+     * 
              
      * Note that the definition should *not* be nested under a type name.
      *
      * @param source The mapping source
@@ -432,7 +432,7 @@ public CreateIndexRequest source(String source, XContentType xContentType) {
 
     /**
      * Sets the settings and mappings as a single source.
-     *
+     * 
              
      * Note that the mapping definition should *not* be nested under a type name.
      */
     public CreateIndexRequest source(String source, MediaType mediaType) {
@@ -458,7 +458,7 @@ public CreateIndexRequest source(byte[] source, XContentType xContentType) {
 
     /**
      * Sets the settings and mappings as a single source.
-     *
+     * 
              
      * Note that the mapping definition should *not* be nested under a type name.
      */
     public CreateIndexRequest source(byte[] source, MediaType mediaType) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
index 0cff01af536dc..384ae2e028bba 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
@@ -130,7 +130,7 @@ protected void writeNodesTo(StreamOutput out, List
      * NOTE: visible for testing
      *
      * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
index e3c812aedcfe7..6a1d04efa9714 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
@@ -46,7 +46,7 @@
 
 /**
  * Request the mappings of specific fields
- *
+ * 
              
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should go to that client class as well.
  *
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index d874b5bb6b1ac..94c88e30295a8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -61,7 +61,7 @@
 
 /**
  * Response object for {@link GetFieldMappingsRequest} API
- *
+ * 
              
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should go to that client class as well.
  *
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 4a2d2fbe8e950..32f751ceb1c5a 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -210,7 +210,7 @@ public String source() {
     /**
      * A specialized simplified mapping source method, takes the form of simple properties definition:
      * ("field1", "type=string,store=true").
-     *
+     * 
              
      * Also supports metadata mapping fields such as `_all` and `_parent` as property definition, these metadata
      * mapping fields will automatically be put on the top level mapping object.
      */
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java
index c46940fbfecf9..b82b68f6f9489 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java
@@ -89,7 +89,7 @@ public void activeOnly(boolean activeOnly) {
     /**
      * Contains list of shard id's if shards are passed, empty otherwise. Array is empty by default.
      *
-     * @return  list of shard id's if shards are passed, empty otherwise
+     * @return  array of shard id's if shards are passed, empty otherwise
      */
     public String[] shards() {
         return shards;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
index b25bc94a5c8e2..353cdbbbc840c 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
@@ -54,7 +54,7 @@
 
 /**
  * Request class to swap index under an alias or increment data stream generation upon satisfying conditions
- *
+ * 
              
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should also go to that client class.
  *
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java
index 55ee65d0a4973..a66fcc9e9bcf2 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java
@@ -51,7 +51,7 @@
 
 /**
  * Response object for {@link RolloverRequest} API
- *
+ * 
              
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should also go to that client class.
  *
diff --git a/server/src/main/java/org/opensearch/action/bulk/BackoffPolicy.java b/server/src/main/java/org/opensearch/action/bulk/BackoffPolicy.java
index 0d6d122e31261..25a2c081f8441 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BackoffPolicy.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BackoffPolicy.java
@@ -40,9 +40,9 @@
 /**
  * Provides a backoff policy for bulk requests. Whenever a bulk request is rejected due to resource constraints (i.e. the client's internal
  * thread pool is full), the backoff policy decides how long the bulk processor will wait before the operation is retried internally.
- *
+ * 
              
  * Notes for implementing custom subclasses:
- *
+ * 
              
  * The underlying mathematical principle of BackoffPolicy are progressions which can be either finite or infinite although
  * the latter should not be used for retrying. A progression can be mapped to a java.util.Iterator with the following
  * semantics:
@@ -241,7 +241,7 @@ private static class ExponentialEqualJitterBackoffIterator implements Iterator
          * NOTE: If the value is greater than 30, there can be integer overflow
          * issues during delay calculation.
          **/
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java
index 138f21f1f80e5..20dea872e0ad2 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java
@@ -209,7 +209,7 @@ public static class Failure implements Writeable, ToXContentFragment {
 
         /**
          * For write failures before operation was assigned a sequence number.
-         *
+         * 
              
          * use @{link {@link #Failure(String, String, Exception, long, long)}}
          * to record operation sequence no with failure
          */
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java
index baf64b3e80af6..141ec24fc390f 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java
@@ -185,7 +185,7 @@ public Builder setGlobalPipeline(String globalPipeline) {
         /**
          * Sets a custom backoff policy. The backoff policy defines how the bulk processor should handle retries of bulk requests internally
          * in case they have failed due to resource constraints (i.e. a thread pool was full).
-         *
+         * 
              
          * The default is to back off exponentially.
          *
          * @see org.opensearch.action.bulk.BackoffPolicy#exponentialBackoff()
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
index 65043da6c2684..f2f3077001a13 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
@@ -67,7 +67,7 @@
 /**
  * A bulk request holds an ordered {@link IndexRequest}s, {@link DeleteRequest}s and {@link UpdateRequest}s
  * and allows to executes it in a single batch.
- *
+ * 
              
  * Note that we only support refresh on the bulk request not per item.
  * @see org.opensearch.client.Client#bulk(BulkRequest)
  *
@@ -123,7 +123,7 @@ public BulkRequest add(DocWriteRequest... requests) {
 
     /**
      * Add a request to the current BulkRequest.
-     *
+     * 
              
      * Note for internal callers: This method does not respect all global parameters.
      *                            Only the global index is applied to the request objects.
      *                            Global parameters would be respected if the request was serialized for a REST call as it is
@@ -347,7 +347,7 @@ public final BulkRequest timeout(TimeValue timeout) {
     /**
      * Note for internal callers (NOT high level rest client),
      * the global parameter setting is ignored when used with:
-     *
+     * 
              
      * - {@link BulkRequest#add(IndexRequest)}
      * - {@link BulkRequest#add(UpdateRequest)}
      * - {@link BulkRequest#add(DocWriteRequest)}
@@ -364,7 +364,7 @@ public final BulkRequest pipeline(String globalPipeline) {
     /**
      * Note for internal callers (NOT high level rest client),
      * the global parameter setting is ignored when used with:
-     *
+     * 
              
       - {@link BulkRequest#add(IndexRequest)}
       - {@link BulkRequest#add(UpdateRequest)}
       - {@link BulkRequest#add(DocWriteRequest)}
@@ -404,7 +404,7 @@ public Boolean requireAlias() {
     /**
      * Note for internal callers (NOT high level rest client),
      * the global parameter setting is ignored when used with:
-     *
+     * 
              
      * - {@link BulkRequest#add(IndexRequest)}
      * - {@link BulkRequest#add(UpdateRequest)}
      * - {@link BulkRequest#add(DocWriteRequest)}
diff --git a/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java b/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java
index 81ae3d78e8ced..9a9e861ad8055 100644
--- a/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java
+++ b/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java
@@ -205,7 +205,7 @@ public long ifSeqNo() {
 
     /**
      * If set, only perform this delete request if the document was last modification was assigned this primary term.
-     *
+     * 
              
      * If the document last modification was assigned a different term a
      * {@link org.opensearch.index.engine.VersionConflictEngineException} will be thrown.
      */
diff --git a/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java b/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java
index 039214459ac21..6cbabfec6d763 100644
--- a/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java
+++ b/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java
@@ -40,7 +40,7 @@
 
 /**
  * Performs the delete operation.
- *
+ * 
              
  * Deprecated use TransportBulkAction with a single item instead
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequest.java b/server/src/main/java/org/opensearch/action/index/IndexRequest.java
index d0b78cfd49d4b..c7926e65cf792 100644
--- a/server/src/main/java/org/opensearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/index/IndexRequest.java
@@ -76,14 +76,14 @@
 /**
  * Index request to index a typed JSON document into a specific index and make it searchable. Best
  * created using {@link org.opensearch.client.Requests#indexRequest(String)}.
- *
+ * 
              
  * The index requires the {@link #index()}, {@link #id(String)} and
  * {@link #source(byte[], MediaType)} to be set.
- *
+ * 
              
  * The source (content to index) can be set in its bytes form using ({@link #source(byte[], MediaType)}),
  * its string form ({@link #source(String, MediaType)}) or using a {@link XContentBuilder}
  * ({@link #source(XContentBuilder)}).
- *
+ * 
              
  * If the {@link #id(String)} is not set, it will be automatically generated.
  *
  * @see IndexResponse
@@ -388,7 +388,7 @@ public IndexRequest source(Map source, MediaType contentType) throws
 
     /**
      * Sets the document source to index.
-     *
+     * 
              
      * Note, its preferable to either set it using {@link #source(XContentBuilder)}
      * or using the {@link #source(byte[], MediaType)}.
      */
@@ -591,7 +591,7 @@ public long ifSeqNo() {
 
     /**
      * If set, only perform this indexing request if the document was last modification was assigned this primary term.
-     *
+     * 
              
      * If the document last modification was assigned a different term a
      * {@link org.opensearch.index.engine.VersionConflictEngineException} will be thrown.
      */
diff --git a/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java b/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java
index fe4f80bf0c065..ce32840f6751b 100644
--- a/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java
+++ b/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java
@@ -40,7 +40,7 @@
 
 /**
  * Performs the index operation.
- *
+ * 
              
  * Allows for the following settings:
  * 
                
  * 
              • autoCreateIndex: When set to {@code true}, will automatically create an index if one does not exists.
diff --git a/server/src/main/java/org/opensearch/action/ingest/IngestActionForwarder.java b/server/src/main/java/org/opensearch/action/ingest/IngestActionForwarder.java
index 9927affbc7442..2821f4fd7fadb 100644
--- a/server/src/main/java/org/opensearch/action/ingest/IngestActionForwarder.java
+++ b/server/src/main/java/org/opensearch/action/ingest/IngestActionForwarder.java
@@ -46,7 +46,7 @@
 
 /**
  * A utility for forwarding ingest requests to ingest nodes in a round-robin fashion.
- *
+ * 
                
  * TODO: move this into IngestService and make index/bulk actions call that
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitController.java b/server/src/main/java/org/opensearch/action/search/CreatePitController.java
index 1d11ad2760675..87eb27bdb8255 100644
--- a/server/src/main/java/org/opensearch/action/search/CreatePitController.java
+++ b/server/src/main/java/org/opensearch/action/search/CreatePitController.java
@@ -98,18 +98,18 @@ public void executeCreatePit(
             task.getParentTaskId(),
             Collections.emptyMap()
         );
-        /**
-         * This is needed for cross cluster functionality to work with PITs and current ccsMinimizeRoundTrips is
-         * not supported for point in time
+        /*
+          This is needed for cross cluster functionality to work with PITs and current ccsMinimizeRoundTrips is
+          not supported for point in time
          */
         searchRequest.setCcsMinimizeRoundtrips(false);
-        /**
-         * Phase 1 of create PIT
+        /*
+          Phase 1 of create PIT
          */
         executeCreatePit(searchTask, searchRequest, createPitListener);
 
-        /**
-         * Phase 2 of create PIT where we update pit id in pit contexts
+        /*
+          Phase 2 of create PIT where we update pit id in pit contexts
          */
         createPitListener.whenComplete(
             searchResponse -> { executeUpdatePitId(request, searchRequest, searchResponse, updatePitIdListener); },
@@ -167,9 +167,9 @@ void executeUpdatePitId(
                 searchResponse.pointInTimeId()
             )
         );
-        /**
-         * store the create time ( same create time for all PIT contexts across shards ) to be used
-         * for list PIT api
+        /*
+          store the create time ( same create time for all PIT contexts across shards ) to be used
+          for list PIT api
          */
         final long relativeStartNanos = System.nanoTime();
         final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(
diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java
index cca85f92d2676..161a103cdf36a 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java
@@ -162,7 +162,7 @@ public AggregatedDfs aggregateDfs(Collection results) {
      * Returns a score doc array of top N search docs across all shards, followed by top suggest docs for each
      * named completion suggestion across all shards. If more than one named completion suggestion is specified in the
      * request, the suggest docs for a named suggestion are ordered by the suggestion name.
-     *
+     * 
                
      * Note: The order of the sorted score docs depends on the shard index in the result array if the merge process needs to disambiguate
      * the result. In oder to obtain stable results the shard index (index of the result in the result array) must be the same.
      *
@@ -284,7 +284,7 @@ public List[] fillDocIdsToLoad(int numShards, ScoreDoc[] shardDocs) {
     /**
      * Enriches search hits and completion suggestion hits from sortedDocs using fetchResultsArr,
      * merges suggestions, aggregations and profile results
-     *
+     * 
                
      * Expects sortedDocs to have top search docs across all shards, optionally followed by top suggest docs for each named
      * completion suggestion ordered by suggestion name
      */
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
index 762022460ebdb..21cf0ed97b9da 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
@@ -600,7 +600,7 @@ public void setMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
      * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
      * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
      * bounds and the query are disjoint.
-     *
+     * 
                
      * When unspecified, the pre-filter phase is executed if any of these conditions is met:
      * 
                  
      * 
                • The request targets more than 128 shards
                  • 
@@ -621,7 +621,7 @@ public void setPreFilterShardSize(int preFilterShardSize) {
      * This filter roundtrip can limit the number of shards significantly if for
      * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
      * bounds and the query are disjoint.
-     *
+     * 
                  
      * When unspecified, the pre-filter phase is executed if any of these conditions is met:
      * 
                    
      * 
                  • The request targets more than 128 shards
                    • 
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
index 861e1df0203d7..bc43b65e5d844 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
@@ -605,7 +605,7 @@ public SearchRequestBuilder setMaxConcurrentShardRequests(int maxConcurrentShard
      * the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for
      * instance a shard can not match any documents based on its rewrite method ie. if date filters are mandatory to match but the shard
      * bounds and the query are disjoint.
-     *
+     * 
                    
      * When unspecified, the pre-filter phase is executed if any of these conditions is met:
      * 
                      
      * 
                    • The request targets more than 128 shards
                      • 
diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java
index 2e447abd125c5..579a01f0dd932 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java
@@ -53,7 +53,7 @@
 /**
  * Base class that holds the various sections which a search response is
  * composed of (hits, aggs, suggestions etc.) and allows to retrieve them.
- *
+ * 
                      
  * The reason why this class exists is that the high level REST client uses its own classes
  * to parse aggregations into, which are not serializable. This is the common part that can be
  * shared between core and client.
diff --git a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java
index 614d576324026..b15a4b66e8870 100644
--- a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java
+++ b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java
@@ -76,7 +76,7 @@ private void deletePits(ActionListener listener, DeletePitReq
 
     /**
      * Delete all active PIT reader contexts leveraging list all PITs
-     *
+     * 
                      
      * For Cross cluster PITs :
      * - mixed cluster PITs ( PIT comprising local and remote ) will be fully deleted. Since there will atleast be
      * one reader context with PIT ID present in local cluster, 'Get all PITs' will retrieve the PIT ID with which
diff --git a/server/src/main/java/org/opensearch/action/support/TransportAction.java b/server/src/main/java/org/opensearch/action/support/TransportAction.java
index daa11c2d7d80f..72aae210d61ae 100644
--- a/server/src/main/java/org/opensearch/action/support/TransportAction.java
+++ b/server/src/main/java/org/opensearch/action/support/TransportAction.java
@@ -81,7 +81,7 @@ private Releasable registerChildNode(TaskId parentTask) {
 
     /**
      * Use this method when the transport action call should result in creation of a new task associated with the call.
-     *
+     * 
                      
      * This is a typical behavior.
      */
     public final Task execute(Request request, ActionListener listener) {
diff --git a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java
index f5fb41dc5bae3..4d54ce51c923c 100644
--- a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java
+++ b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodesRequest.java
@@ -53,9 +53,9 @@ public abstract class BaseNodesRequest
      * the list of nodesIds that will be used to resolve this request and {@link #concreteNodes}
      * will be populated. Note that if {@link #concreteNodes} is not null, it will be used and nodeIds
      * will be ignored.
-     *
+     * 
                      
      * See {@link DiscoveryNodes#resolveNodes} for a full description of the options.
-     *
+     * 
                      
      * TODO: get rid of this and resolve it to concrete nodes in the rest layer
      **/
     private String[] nodesIds;
diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java
index 60c490a50575a..9f69d41d83f5b 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java
@@ -479,7 +479,7 @@ public interface Primary<
 
         /**
          * Notifies the primary of a local checkpoint for the given allocation.
-         *
+         * 
                      
          * Note: The primary will use this information to advance the global checkpoint if possible.
          *
          * @param allocationId allocation ID of the shard corresponding to the supplied local checkpoint
diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
index de5a92fdcc4b1..b68bd13cfed80 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
@@ -100,7 +100,7 @@
 /**
  * Base class for requests that should be executed on a primary copy followed by replica copies.
  * Subclasses can resolve the target shard and provide implementation for primary and replica operations.
- *
+ * 
                      
  * The action samples cluster state on the receiving node to reroute to node with primary copy and on the
  * primary node to validate request before primary operation followed by sampling state again for resolving
  * nodes with replica copies to perform replication.
@@ -866,7 +866,7 @@ protected IndexShard getIndexShard(final ShardId shardId) {
      * Responsible for routing and retrying failed operations on the primary.
      * The actual primary operation is done in {@link ReplicationOperation} on the
      * node with primary copy.
-     *
+     * 
                      
      * Resolves index and shard id for the request before routing it to target node
      *
      * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java
index 62cbfbde9780a..a0b5299805868 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java
@@ -266,7 +266,7 @@ protected abstract void dispatchedShardOperationOnReplica(
 
     /**
      * Result of taking the action on the primary.
-     *
+     * 
                      
      * NOTE: public for testing
      *
      * @opensearch.internal
@@ -496,7 +496,7 @@ void run() {
      * A proxy for write operations that need to be performed on the
      * replicas, where a failure to execute the operation should fail
      * the replica shard and/or mark the replica as stale.
-     *
+     * 
                      
      * This extends {@code TransportReplicationAction.ReplicasProxy} to do the
      * failing and stale-ing.
      *
diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java b/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java
index c474096ff94e4..56b34aea8248d 100644
--- a/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java
+++ b/server/src/main/java/org/opensearch/action/support/single/shard/SingleShardRequest.java
@@ -55,7 +55,7 @@ public abstract class SingleShardRequest
      * Whether index property is optional depends on the concrete implementation. If index property is required the
      * concrete implementation should use {@link #validateNonNullIndex()} to check if the index property has been set
      */
diff --git a/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java b/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java
index a10f74f7c2aa8..c2ae333b17055 100644
--- a/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java
+++ b/server/src/main/java/org/opensearch/action/support/tasks/BaseTasksRequest.java
@@ -128,7 +128,7 @@ public final Request setNodes(String... nodes) {
 
     /**
      * Returns the id of the task that should be processed.
-     *
+     * 
                      
      * By default tasks with any ids are returned.
      */
     public TaskId getTaskId() {
diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
index b354482871521..946c3c2446173 100644
--- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
+++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
@@ -589,7 +589,7 @@ public long ifSeqNo() {
 
     /**
      * If set, only perform this update request if the document was last modification was assigned this primary term.
-     *
+     * 
                      
      * If the document last modification was assigned a different term a
      * {@link org.opensearch.index.engine.VersionConflictEngineException} will be thrown.
      */
diff --git a/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java
index f9661e71d60e6..e43c42446de2c 100644
--- a/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java
@@ -73,7 +73,7 @@
 
 /**
  * We enforce bootstrap checks once a node has the transport protocol bound to a non-loopback interface or if the system property {@code
- * opensearch.enforce.bootstrap.checks} is set to {@true}. In this case we assume the node is running in production and
+ * opensearch.enforce.bootstrap.checks} is set to {@code true}. In this case we assume the node is running in production and
  * all bootstrap checks must pass.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/bootstrap/JNAKernel32Library.java b/server/src/main/java/org/opensearch/bootstrap/JNAKernel32Library.java
index 8e556df4b2f9b..91da34fb7216d 100644
--- a/server/src/main/java/org/opensearch/bootstrap/JNAKernel32Library.java
+++ b/server/src/main/java/org/opensearch/bootstrap/JNAKernel32Library.java
@@ -141,7 +141,7 @@ public boolean callback(long dwCtrlType) {
 
     /**
      * Memory protection constraints
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366786%28v=vs.85%29.aspx
      */
     public static final int PAGE_NOACCESS = 0x0001;
@@ -151,7 +151,7 @@ public boolean callback(long dwCtrlType) {
     /**
      * Contains information about a range of pages in the virtual address space of a process.
      * The VirtualQuery and VirtualQueryEx functions use this structure.
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366775%28v=vs.85%29.aspx
      */
     public static class MemoryBasicInformation extends Structure {
@@ -186,7 +186,7 @@ public SizeT() {
     /**
      * Locks the specified region of the process's virtual address space into physical
      * memory, ensuring that subsequent access to the region will not incur a page fault.
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366895%28v=vs.85%29.aspx
      *
      * @param address A pointer to the base address of the region of pages to be locked.
@@ -197,7 +197,7 @@ public SizeT() {
 
     /**
      * Retrieves information about a range of pages within the virtual address space of a specified process.
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366907%28v=vs.85%29.aspx
      *
      * @param handle A handle to the process whose memory information is queried.
@@ -210,7 +210,7 @@ public SizeT() {
 
     /**
      * Sets the minimum and maximum working set sizes for the specified process.
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms686234%28v=vs.85%29.aspx
      *
      * @param handle A handle to the process whose working set sizes is to be set.
@@ -222,7 +222,7 @@ public SizeT() {
 
     /**
      * Retrieves a pseudo handle for the current process.
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms683179%28v=vs.85%29.aspx
      *
      * @return a pseudo handle to the current process.
@@ -231,7 +231,7 @@ public SizeT() {
 
     /**
      * Closes an open object handle.
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx
      *
      * @param handle A valid handle to an open object.
@@ -252,7 +252,7 @@ public SizeT() {
 
     /**
      * Creates or opens a new job object
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms682409%28v=vs.85%29.aspx
      *
      * @param jobAttributes security attributes
@@ -263,7 +263,7 @@ public SizeT() {
 
     /**
      * Associates a process with an existing job
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms681949%28v=vs.85%29.aspx
      *
      * @param job job handle
@@ -274,7 +274,7 @@ public SizeT() {
 
     /**
      * Basic limit information for a job object
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms684147%28v=vs.85%29.aspx
      */
     public static class JOBOBJECT_BASIC_LIMIT_INFORMATION extends Structure implements Structure.ByReference {
@@ -316,7 +316,7 @@ protected List getFieldOrder() {
 
     /**
      * Get job limit and state information
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms684925%28v=vs.85%29.aspx
      *
      * @param job job handle
@@ -330,7 +330,7 @@ protected List getFieldOrder() {
 
     /**
      * Set job limit and state information
-     *
+     * 
                      
      * https://msdn.microsoft.com/en-us/library/windows/desktop/ms686216%28v=vs.85%29.aspx
      *
      * @param job job handle
diff --git a/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java b/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java
index ab52ae5a43a2a..4d36efff0e192 100644
--- a/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java
+++ b/server/src/main/java/org/opensearch/bootstrap/OpenSearch.java
@@ -188,9 +188,9 @@ void init(final boolean daemonize, final Path pidFile, final boolean quiet, Envi
     /**
      * Required method that's called by Apache Commons procrun when
      * running as a service on Windows, when the service is stopped.
-     *
+     * 
                      
      * http://commons.apache.org/proper/commons-daemon/procrun.html
-     *
+     * 
                      
      * NOTE: If this method is renamed and/or moved, make sure to
      * update opensearch-service.bat!
      */
diff --git a/server/src/main/java/org/opensearch/cluster/AckedClusterStateTaskListener.java b/server/src/main/java/org/opensearch/cluster/AckedClusterStateTaskListener.java
index 482087be1c8eb..28e1e7c53cb9c 100644
--- a/server/src/main/java/org/opensearch/cluster/AckedClusterStateTaskListener.java
+++ b/server/src/main/java/org/opensearch/cluster/AckedClusterStateTaskListener.java
@@ -44,7 +44,7 @@ public interface AckedClusterStateTaskListener extends ClusterStateTaskListener
 
     /**
      * Called to determine which nodes the acknowledgement is expected from.
-     *
+     * 
                      
      * As this method will be called multiple times to determine the set of acking nodes,
      * it is crucial for it to return consistent results: Given the same listener instance
      * and the same node parameter, the method implementation should return the same result.
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java
index 50beeb1f03deb..bf8494cc36857 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java
@@ -70,7 +70,7 @@ default boolean runOnlyOnMaster() {
     /**
      * Callback invoked after new cluster state is published. Note that
      * this method is not invoked if the cluster state was not updated.
-     *
+     * 
                      
      * Note that this method will be executed using system context.
      *
      * @param clusterChangedEvent the change event for this cluster state change, containing
@@ -80,7 +80,7 @@ default void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {}
 
     /**
      * Builds a concise description of a list of tasks (to be used in logging etc.).
-     *
+     * 
                      
      * Note that the tasks given are not necessarily the same as those that will be passed to {@link #execute(ClusterState, List)}.
      * but are guaranteed to be a subset of them. This method can be called multiple times with different lists before execution.
      * This allows groupd task description but the submitting source.
diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java
index dd2232968114e..a38fc81bebc08 100644
--- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java
+++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java
@@ -182,7 +182,7 @@ public Map apply(Map map) {
 
     /**
      * Represents differences between two maps of objects and is used as base class for different map implementations.
-     *
+     * 
                      
      * Implements serialization. How differences are applied is left to subclasses.
      *
      * @param  the type of map keys
@@ -381,9 +381,9 @@ public Integer readKey(StreamInput in) throws IOException {
     /**
      * Provides read and write operations to serialize map values.
      * Reading of values can be made dependent on map key.
-     *
+     * 
                      
      * Also provides operations to distinguish whether map values are diffable.
-     *
+     * 
                      
      * Should not be directly implemented, instead implement either
      * {@link DiffableValueSerializer} or {@link NonDiffableValueSerializer}.
      *
@@ -517,7 +517,7 @@ public Diff readDiff(StreamInput in, K key) throws IOException {
 
     /**
      * Implementation of the ValueSerializer that wraps value and diff readers.
-     *
+     * 
                      
      * Note: this implementation is ignoring the key.
      *
      * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java
index 35490d2f37a49..e381b8f244bf3 100644
--- a/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java
+++ b/server/src/main/java/org/opensearch/cluster/InternalClusterInfoService.java
@@ -82,7 +82,7 @@
  * to 30 seconds). The InternalClusterInfoService only runs on the cluster-manager node.
  * Listens for changes in the number of data nodes and immediately submits a
  * ClusterInfoUpdateJob if a node has been added.
- *
+ * 
                      
  * Every time the timer runs, gathers information about the disk usage and
  * shard sizes across the cluster.
  *
diff --git a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java
index 042a4743ca25d..3e0c78099e6b4 100644
--- a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java
+++ b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java
@@ -189,7 +189,7 @@ public Snapshot snapshot() {
         /**
          * Returns list of shards that being restore and their status
          *
-         * @return list of shards
+         * @return map of shard id to shard restore status
          */
         public Map shards() {
             return this.shards;
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java b/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java
index 39d05e672f977..3a506397bcac8 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java
@@ -46,11 +46,11 @@ public interface ClusterStatePublisher {
     /**
      * Publish all the changes to the cluster from the cluster-manager (can be called just by the cluster-manager). The publish
      * process should apply this state to the cluster-manager as well!
-     *
+     * 
                      
      * The publishListener allows to wait for the publication to complete, which can be either successful completion, timing out or failing.
      * The method is guaranteed to pass back a {@link FailedToCommitClusterStateException} to the publishListener if the change is not
      * committed and should be rejected. Any other exception signals that something bad happened but the change is committed.
-     *
+     * 
                      
      * The {@link AckListener} allows to keep track of the ack received from nodes, and verify whether
      * they updated their own cluster state or not.
      */
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
index 15eaf9c8bcc1e..f701a2f52277d 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
@@ -483,10 +483,10 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata)
      * 2. The joining node has to be a non-remote store backed if it is joining a non-remote store backed cluster.
      * Validates no remote store attributes are present in joining node as existing nodes in the cluster doesn't have
      * remote store attributes.
-     *
+     * 
                      
      * A remote store backed node is the one which holds all the remote store attributes and a remote store backed
      * cluster is the one which has only homogeneous remote store backed nodes with same node attributes
-     *
+     * 
                      
      * TODO: When we support moving from remote store cluster to non remote store and vice versa the this logic will
      *       needs to be modified.
      */
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Reconfigurator.java b/server/src/main/java/org/opensearch/cluster/coordination/Reconfigurator.java
index 1570a84ab871f..128bd42fd7947 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/Reconfigurator.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/Reconfigurator.java
@@ -63,7 +63,7 @@ public class Reconfigurator {
      * and still the cluster would be unavailable. However not reducing the voting configuration size can also hamper resilience: in a
      * five-node cluster we could lose two nodes and by reducing the voting configuration to the remaining three nodes we could tolerate the
      * loss of a further node before failing.
-     *
+     * 
                      
      * We offer two options: either we auto-shrink the voting configuration as long as it contains more than three nodes, or we don't and we
      * require the user to control the voting configuration manually using the retirement API. The former, default, option, guarantees that
      * as long as there have been at least three cluster-manager-eligible nodes in the cluster and no more than one of them is currently unavailable,
diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java
index d06e89d9ea170..cbc63565228f9 100644
--- a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java
+++ b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java
@@ -219,13 +219,13 @@ public void writeTo(final StreamOutput out) throws IOException {
 
     /**
      * Checks if an inactive primary shard should cause the cluster health to go RED.
-     *
+     * 
                      
      * An inactive primary shard in an index should cause the cluster health to be RED to make it visible that some of the existing data is
      * unavailable. In case of index creation, snapshot restore or index shrinking, which are unexceptional events in the cluster lifecycle,
      * cluster health should not turn RED for the time where primaries are still in the initializing state but go to YELLOW instead.
      * However, in case of exceptional events, for example when the primary shard cannot be assigned to a node or initialization fails at
      * some point, cluster health should still turn RED.
-     *
+     * 
                      
      * NB: this method should *not* be called on active shards nor on non-primary shards.
      */
     public static ClusterHealthStatus getInactivePrimaryHealth(final ShardRouting shardRouting) {
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java
index 0da948dc78c5d..85a203e5e059a 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java
@@ -61,7 +61,7 @@
 
 /**
  * A collection of tombstones for explicitly marking indices as deleted in the cluster state.
- *
+ * 
                      
  * The cluster state contains a list of index tombstones for indices that have been
  * deleted in the cluster.  Because cluster states are processed asynchronously by
  * nodes and a node could be removed from the cluster for a period of time, the
@@ -250,7 +250,7 @@ public int getNumPurged() {
 
         /**
          * Purge tombstone entries.  Returns the number of entries that were purged.
-         *
+         * 
                      
          * Tombstones are purged if the number of tombstones in the list
          * is greater than the input parameter of maximum allowed tombstones.
          * Tombstones are purged until the list is equal to the maximum allowed.
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
index 52df72b342b3e..5d8b751b241e2 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -779,7 +779,7 @@ public long getAliasesVersion() {
     /**
      * The term of the current selected primary. This is a non-negative number incremented when
      * a primary shard is assigned after a full cluster restart or a replica shard is promoted to a primary.
-     *
+     * 
                      
      * Note: since we increment the term every time a shard is assigned, the term for any operational shard (i.e., a shard
      * that can be indexed into) is larger than 0. See {@link IndexMetadataUpdater#applyChanges}.
      **/
@@ -1884,7 +1884,7 @@ public static Settings addHumanReadableSettings(Settings settings) {
 
     /**
      * Return the version the index was created from the provided index settings
-     *
+     * 
                      
      * This looks for the presence of the {@link Version} object with key {@link IndexMetadata#SETTING_VERSION_CREATED}
      */
     public static Version indexCreated(final Settings indexSettings) {
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java
index 272bb132197af..c5efb55316b84 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java
@@ -369,7 +369,7 @@ public IndexTemplateMetadata build() {
         /**
          * Serializes the template to xContent, using the legacy format where the mappings are
          * nested under the type name.
-         *
+         * 
                      
          * This method is used for serializing templates before storing them in the cluster metadata,
          * and also in the REST layer when returning a deprecated typed response.
          */
@@ -386,7 +386,7 @@ public static void toXContentWithTypes(
         /**
          * Serializes the template to xContent, making sure not to nest mappings under the
          * type name.
-         *
+         * 
                      
          * Note that this method should currently only be used for creating REST responses,
          * and not when directly updating stored templates. Index templates are still stored
          * in the old, typed format, and have yet to be migrated to be typeless.
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
index 146193b8d22c4..626903877b0c6 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
@@ -1666,7 +1666,7 @@ private SortedMap buildIndicesLookup() {
 
         /**
          * Validates there isn't any index with a name that would clash with the future backing indices of the existing data streams.
-         *
+         * 
                      
          * E.g., if data stream `foo` has backing indices [`.ds-foo-000001`, `.ds-foo-000002`] and the indices lookup contains indices
          * `.ds-foo-000001`, `.ds-foo-000002` and `.ds-foo-000006` this will throw an IllegalStateException (as attempting to rollover the
          * `foo` data stream from generation 5 to 6 will not be possible)
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index b2861e566dd4b..8d76a39712ee3 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -754,7 +754,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(
     /**
      * Parses the provided mappings json and the inheritable mappings from the templates (if any)
      * into a map.
-     *
+     * 
                      
      * The template mappings are applied in the order they are encountered in the list (clients
      * should make sure the lower index, closer to the head of the list, templates have the highest
      * {@link IndexTemplateMetadata#order()}). This merging makes no distinction between field
@@ -792,7 +792,7 @@ static Map parseV1Mappings(
      * Validates and creates the settings for the new index based on the explicitly configured settings via the
      * {@link CreateIndexClusterStateUpdateRequest}, inherited from templates and, if recovering from another index (ie. split, shrink,
      * clone), the resize settings.
-     *
+     * 
                      
      * The template mappings are applied in the order they are encountered in the list (clients should make sure the lower index, closer
      * to the head of the list, templates have the highest {@link IndexTemplateMetadata#order()})
      *
@@ -1009,7 +1009,7 @@ static int getIndexNumberOfRoutingShards(Settings indexSettings, @Nullable Index
     /**
      * Validate and resolve the aliases explicitly set for the index, together with the ones inherited from the specified
      * templates.
-     *
+     * 
                      
      * The template mappings are applied in the order they are encountered in the list (clients should make sure the lower index, closer
      * to the head of the list, templates have the highest {@link IndexTemplateMetadata#order()})
      *
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java
index 91c996448ea8f..e30e878f1b31a 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexStateService.java
@@ -166,7 +166,7 @@ public MetadataIndexStateService(
 
     /**
      * Closes one or more indices.
-     *
+     * 
                      
      * Closing indices is a 3 steps process: it first adds a write block to every indices to close, then waits for the operations on shards
      * to be terminated and finally closes the indices by moving their state to CLOSE.
      */
@@ -302,7 +302,7 @@ public TimeValue timeout() {
 
     /**
      * Step 1 - Start closing indices by adding a write block
-     *
+     * 
                      
      * This step builds the list of indices to close (the ones explicitly requested that are not in CLOSE state) and adds a unique cluster
      * block (or reuses an existing one) to every index to close in the cluster state. After the cluster state is published, the shards
      * should start to reject writing operations and we can proceed with step 2.
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
index 71b86ec853ce4..1093ac09777e7 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -747,7 +747,7 @@ public static Map> findConflictingV2Templates(
     /**
      * Return a map of v2 template names to their index patterns for v2 templates that would overlap
      * with the given template's index patterns.
-     *
+     * 
                      
      * Based on the provided checkPriority and priority parameters this aims to report the overlapping
      * index templates regardless of the priority (ie. checkPriority == false) or otherwise overlapping
      * templates with the same priority as the given priority parameter (this is useful when trying to
diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java
index 675d4e09e3be0..561f1b727efe0 100644
--- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java
+++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java
@@ -356,7 +356,7 @@ public DiscoveryNode findByAddress(TransportAddress address) {
 
     /**
      * Returns the version of the node with the oldest version in the cluster that is not a client node
-     *
+     * 
                      
      * If there are no non-client nodes, Version.CURRENT will be returned.
      *
      * @return the oldest version in the cluster
@@ -367,7 +367,7 @@ public Version getSmallestNonClientNodeVersion() {
 
     /**
      * Returns the version of the node with the youngest version in the cluster that is not a client node.
-     *
+     * 
                      
      * If there are no non-client nodes, Version.CURRENT will be returned.
      *
      * @return the youngest version in the cluster
@@ -417,16 +417,16 @@ public DiscoveryNode resolveNode(String node) {
     /**
      * Resolves a set of nodes according to the given sequence of node specifications. Implements the logic in various APIs that allow the
      * user to run the action on a subset of the nodes in the cluster. See [Node specification] in the reference manual for full details.
-     *
+     * 
                      
      * Works by tracking the current set of nodes and applying each node specification in sequence. The set starts out empty and each node
      * specification may either add or remove nodes. For instance:
-     *
+     * 
                      
      * - _local, _cluster_manager (_master) and _all respectively add to the subset the local node, the currently-elected cluster_manager, and all the nodes
      * - node IDs, names, hostnames and IP addresses all add to the subset any nodes which match
      * - a wildcard-based pattern of the form "attr*:value*" adds to the subset all nodes with a matching attribute with a matching value
      * - role:true adds to the subset all nodes with a matching role
      * - role:false removes from the subset all nodes with a matching role.
-     *
+     * 
                      
      * An empty sequence of node specifications returns all nodes, since the corresponding actions run on all nodes by default.
      */
     public String[] resolveNodes(String... nodes) {
@@ -813,7 +813,7 @@ public Builder localNodeId(String localNodeId) {
          * Checks that a node can be safely added to this node collection.
          *
          * @return null if all is OK or an error message explaining why a node can not be added.
-         *
+         * 
                      
          * Note: if this method returns a non-null value, calling {@link #add(DiscoveryNode)} will fail with an
          * exception
          */
diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java
index 9cc09c6e4c31c..2dd57431d0375 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java
@@ -463,7 +463,7 @@ private static Map rankNodes(
      * OpenSearch, however, we do not have that sort of broadcast-to-all behavior. In order to prevent a node that gets a high score and
      * then never gets any more requests, we must ensure it eventually returns to a more normal score and can be a candidate for serving
      * requests.
-     *
+     * 
                      
      * This adjustment takes the "winning" node's statistics and adds the average of those statistics with each non-winning node. Let's say
      * the winning node had a queue size of 10 and a non-winning node had a queue of 18. The average queue size is (10 + 18) / 2 = 14 so the
      * non-winning node will have statistics added for a queue size of 14. This is repeated for the response time and service times as well.
diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java
index 5cef46689ffc7..b01e074ce40c2 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java
@@ -49,7 +49,7 @@
 
 /**
  * Represents the recovery source of a shard. Available recovery types are:
- *
+ * 
                      
  * - {@link EmptyStoreRecoverySource} recovery from an empty store
  * - {@link ExistingStoreRecoverySource} recovery from an existing store
  * - {@link PeerRecoverySource} recovery from a primary on another node
diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java
index 4f7b935f15f93..5a4352653cc89 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java
@@ -70,7 +70,7 @@
  * {@link RoutingNodes} represents a copy the routing information contained in the {@link ClusterState cluster state}.
  * It can be either initialized as mutable or immutable (see {@link #RoutingNodes(ClusterState, boolean)}), allowing
  * or disallowing changes to its elements.
- *
+ * 
                      
  * The main methods used to update routing entries are:
  * 
                        
  * 
                      •  {@link #initializeShard} initializes an unassigned shard.
@@ -369,7 +369,7 @@ public ShardRouting activePrimary(ShardId shardId) {
     /**
      * Returns one active replica shard for the given shard id or null if
      * no active replica is found.
-     *
+     * 
                        
      * Since replicas could possibly be on nodes with an older version of OpenSearch than
      * the primary is, this will return replicas on the highest version of OpenSearch when document
      * replication is enabled.
@@ -395,7 +395,7 @@ public ShardRouting activeReplicaWithHighestVersion(ShardId shardId) {
     /**
      * Returns one active replica shard for the given shard id or null if
      * no active replica is found.
-     *
+     * 
                        
      * Since replicas could possibly be on nodes with a higher version of OpenSearch than
      * the primary is, this will return replicas on the oldest version of OpenSearch when segment
      * replication is enabled to allow for replica to read segments from primary.
@@ -544,9 +544,9 @@ public Tuple relocateShard(
 
     /**
      * Applies the relevant logic to start an initializing shard.
-     *
+     * 
                        
      * Moves the initializing shard to started. If the shard is a relocation target, also removes the relocation source.
-     *
+     * 
                        
      * If the started shard is a primary relocation target, this also reinitializes currently initializing replicas as their
      * recovery source changes
      *
@@ -605,9 +605,9 @@ public ShardRouting startShard(Logger logger, ShardRouting initializingShard, Ro
 
     /**
      * Applies the relevant logic to handle a cancelled or failed shard.
-     *
+     * 
                        
      * Moves the shard to unassigned or completely removes the shard (if relocation target).
-     *
+     * 
                        
      * - If shard is a primary, this also fails initializing replicas.
      * - If shard is an active primary, this also promotes an active replica to primary (if such a replica exists).
      * - If shard is a relocating primary, this also removes the primary relocation target shard.
diff --git a/server/src/main/java/org/opensearch/cluster/routing/ShardMovementStrategy.java b/server/src/main/java/org/opensearch/cluster/routing/ShardMovementStrategy.java
index cfdeed5c227b6..7f5109416494e 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/ShardMovementStrategy.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/ShardMovementStrategy.java
@@ -14,7 +14,7 @@
 
 /**
  * ShardMovementStrategy defines the order in which shard movement occurs.
- *
+ * 
                        
  * ShardMovementStrategy values or rather their string representation to be used with
  * {@link BalancedShardsAllocator#SHARD_MOVEMENT_STRATEGY_SETTING} via cluster settings.
  *
diff --git a/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java b/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java
index de36547b10707..5e748df5eed2d 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java
@@ -153,7 +153,7 @@ public enum Reason {
     /**
      * Captures the status of an unsuccessful allocation attempt for the shard,
      * causing it to remain in the unassigned state.
-     *
+     * 
                        
      * Note, ordering of the enum is important, make sure to add new values
      * at the end and handle version serialization properly.
      *
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java
index 19601483d5607..6fc0e535ef4dc 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java
@@ -26,7 +26,7 @@
  * This {@link AwarenessReplicaBalance} gives total unique values of awareness attributes
  * It takes in effect only iff cluster.routing.allocation.awareness.attributes and
  * cluster.routing.allocation.awareness.force.zone.values both are specified.
- *
+ * 
                        
  * This is used in enforcing total copy of shard is a maximum of unique values of awareness attributes
  * Helps in balancing shards across all awareness attributes and ensuring high availability of data.
  */
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java
index f209e993518c1..ae2d4a0926194 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java
@@ -36,14 +36,14 @@ public class ConstraintTypes {
     /**
      * Constraint to control number of shards of an index allocated on a single
      * node.
-     *
+     * 
                        
      * In current weight function implementation, when a node has significantly
      * fewer shards than other nodes (e.g. during single new node addition or node
      * replacement), its weight is much less than other nodes. All shard allocations
      * at this time tend to land on the new node with skewed weight. This breaks
      * index level balance in the cluster, by creating all shards of the same index
      * on one node, often resulting in a hotspot on that node.
-     *
+     * 
                        
      * This constraint is breached when balancer attempts to allocate more than
      * average shards per index per node.
      */
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java
index ae8d92dae6811..7fc78b05880f3 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java
@@ -59,9 +59,9 @@
 /**
  * Observer that tracks changes made to RoutingNodes in order to update the primary terms and in-sync allocation ids in
  * {@link IndexMetadata} once the allocation round has completed.
- *
+ * 
                        
  * Primary terms are updated on primary initialization or when an active primary fails.
- *
+ * 
                        
  * Allocation ids are added for shards that become active and removed for shards that stop being active.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 90eff50fd9b5d..41ace0e7661fe 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -336,7 +336,7 @@ public boolean getPreferPrimaryBalance() {
      * 
                        • 
      * 
                      
      * weight(node, index) = weightindex(node, index) + weightnode(node, index)
-     *
+     * 
                      
      * package-private for testing
      */
     static class WeightFunction {
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
index 3365b58d92a63..75448520a499c 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
@@ -530,7 +530,7 @@ private void checkAndAddInEligibleTargetNode(RoutingNode targetNode) {
 
     /**
      * Move started shards that can not be allocated to a node anymore
-     *
+     * 
                      
      * For each shard to be moved this function executes a move operation
      * to the minimal eligible node with respect to the
      * weight function. If a shard is moved the shard will be set to
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java
index 63d8c656f5049..29e9acca4e6c2 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java
@@ -63,12 +63,12 @@ public interface ShardsAllocator {
      * Returns the decision for where a shard should reside in the cluster.  If the shard is unassigned,
      * then the {@link AllocateUnassignedDecision} will be non-null.  If the shard is not in the unassigned
      * state, then the {@link MoveDecision} will be non-null.
-     *
+     * 
                      
      * This method is primarily used by the cluster allocation explain API to provide detailed explanations
      * for the allocation of a single shard.  Implementations of the {@link #allocate(RoutingAllocation)} method
      * may use the results of this method implementation to decide on allocating shards in the routing table
      * to the cluster.
-     *
+     * 
                      
      * If an implementation of this interface does not support explaining decisions for a single shard through
      * the cluster explain API, then this method should throw a {@code UnsupportedOperationException}.
      */
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java
index 7fffb0299af85..def0411853643 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/command/AllocationCommand.java
@@ -43,7 +43,7 @@
 
 /**
  * A command to move shards in some way.
- *
+ * 
                      
  * Commands are registered in {@link NetworkModule}.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecider.java
index 24c3fd7f34e4a..85f193c8c5580 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AllocationDecider.java
@@ -109,7 +109,7 @@ public Decision canRebalance(RoutingAllocation allocation) {
      * Returns a {@link Decision} whether the given primary shard can be
      * forcibly allocated on the given node. This method should only be called
      * for unassigned primary shards where the node has a shard copy on disk.
-     *
+     * 
                      
      * Note: all implementations that override this behavior should take into account
      * the results of {@link #canAllocate(ShardRouting, RoutingNode, RoutingAllocation)}
      * before making a decision on force allocation, because force allocation should only
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index 1bd47f111591d..2c7df6b81e676 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -73,23 +73,23 @@
 /**
  * The {@link DiskThresholdDecider} checks that the node a shard is potentially
  * being allocated to has enough disk space.
- *
+ * 
                      
  * It has three configurable settings, all of which can be changed dynamically:
- *
+ * 
                      
  * cluster.routing.allocation.disk.watermark.low is the low disk
  * watermark. New shards will not allocated to a node with usage higher than this,
  * although this watermark may be passed by allocating a shard. It defaults to
  * 0.85 (85.0%).
- *
+ * 
                      
  * cluster.routing.allocation.disk.watermark.high is the high disk
  * watermark. If a node has usage higher than this, shards are not allowed to
  * remain on the node. In addition, if allocating a shard to a node causes the
  * node to pass this watermark, it will not be allowed. It defaults to
  * 0.90 (90.0%).
- *
+ * 
                      
  * Both watermark settings are expressed in terms of used disk percentage, or
  * exact byte values for free space (like "500mb")
- *
+ * 
                      
  * cluster.routing.allocation.disk.threshold_enabled is used to
  * enable or disable this decider. It defaults to true (enabled).
  *
@@ -119,7 +119,7 @@ public DiskThresholdDecider(Settings settings, ClusterSettings clusterSettings)
     /**
      * Returns the size of all shards that are currently being relocated to
      * the node, but may not be finished transferring yet.
-     *
+     * 
                      
      * If subtractShardsMovingAway is true then the size of shards moving away is subtracted from the total size of all shards
      */
     public static long sizeOfRelocatingShards(
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
index 1680f2d8cad1d..c2eccdbc6ed26 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
@@ -44,7 +44,7 @@
 /**
  * An allocation decider that prevents multiple instances of the same shard to
  * be allocated on the same {@code node}.
- *
+ * 
                      
  * The {@link #CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING} setting allows to perform a check to prevent
  * allocation of multiple instances of the same shard on a single {@code host},
  * based on host name and host address. Defaults to `false`, meaning that no
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index 3a9fdf0ea10cf..26a04de31ce39 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -323,7 +323,7 @@ private Decision allocateShardCopies(
      * - the initializing shard routing if we want to assign the initializing shard to this node instead
      * - the started shard routing in case if we want to check if we can relocate to this node.
      * - the relocating shard routing if we want to relocate to this node now instead.
-     *
+     * 
                      
      * This method returns the corresponding initializing shard that would be allocated to this node.
      */
     private ShardRouting initializingShard(ShardRouting shardRouting, String currentNodeId) {
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
index 007508162ba14..8e94e7cab23d3 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
@@ -294,7 +294,7 @@ public void addLocalNodeMasterListener(LocalNodeMasterListener listener) {
     /**
      * Adds a cluster state listener that is expected to be removed during a short period of time.
      * If provided, the listener will be notified once a specific time has elapsed.
-     *
+     * 
                      
      * NOTE: the listener is not removed on timeout. This is the responsibility of the caller.
      */
     public void addTimeoutListener(@Nullable final TimeValue timeout, final TimeoutClusterStateListener listener) {
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java
index 8da6b1b941f83..afc4e36ec352e 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java
@@ -29,7 +29,7 @@
 /**
  * This class does throttling on task submission to cluster manager node, it uses throttling key defined in various executors
  * as key for throttling. Throttling will be performed over task executor's class level, different task types have different executors class.
- *
+ * 
                      
  * Set specific setting to for setting the threshold of throttling of particular task type.
  * e.g : Set "cluster_manager.throttling.thresholds.put_mapping" to set throttling limit of "put mapping" tasks,
  *       Set it to default value(-1) to disable the throttling for this task type.
@@ -117,9 +117,9 @@ public static TimeValue getMaxDelayForRetry() {
      * * Register task to cluster service with task key,
      * * override getClusterManagerThrottlingKey method with above task key in task executor.
      * * Verify that throttled tasks would be retried from data nodes
-     *
+     * 
                      
      * Added retry mechanism in TransportClusterManagerNodeAction, so it would be retried for customer generated tasks.
-     *
+     * 
                      
      * If tasks are not getting retried then we can register with false flag, so user won't be able to configure threshold limits for it.
      */
     protected ThrottlingKey registerClusterManagerTask(String taskKey, boolean throttlingEnabled) {
@@ -236,7 +236,7 @@ public void onBeginSubmit(List tasks) {
      * It may start throwing throttling exception to older nodes in cluster.
      * Older version nodes will not be equipped to handle the throttling exception and
      * this may result in unexpected behavior where internal tasks would start failing without any retries.
-     *
+     * 
                      
      * For every task submission request, it will validate if nodes version is greater or equal to 2.5.0 and set the startThrottling flag.
      * Once the startThrottling flag is set, it will not perform check for next set of tasks.
      */
diff --git a/server/src/main/java/org/opensearch/common/Randomness.java b/server/src/main/java/org/opensearch/common/Randomness.java
index 2c60e848b9db9..221bc95c41f31 100644
--- a/server/src/main/java/org/opensearch/common/Randomness.java
+++ b/server/src/main/java/org/opensearch/common/Randomness.java
@@ -127,7 +127,7 @@ public static Random get() {
 
     /**
      * Provides a secure source of randomness.
-     *
+     * 
                      
      * This acts exactly similar to {@link #get()}, but returning a new {@link SecureRandom}.
      */
     public static SecureRandom createSecure() {
diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java
index 3cdb1ce30b68d..2e25a532b5abf 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java
@@ -93,10 +93,10 @@ public interface BlobContainer {
 
     /**
      * Provides a hint to clients for a suitable length to use with {@link BlobContainer#readBlob(String, long, long)}.
-     *
+     * 
                      
      * Some blob containers have nontrivial costs attached to each readBlob call, so it is a good idea for consumers to speculatively
      * request more data than they need right now and to re-use this stream for future needs if possible.
-     *
+     * 
                      
      * Also, some blob containers return streams that are expensive to close before the stream has been fully consumed, and the cost may
      * depend on the length of the data that was left unconsumed. For these containers it's best to bound the cost of a partial read by
      * bounding the length of the data requested.
@@ -131,7 +131,7 @@ default long readBlobPreferredLength() {
     /**
      * Reads blob content from the input stream and writes it to the container in a new blob with the given name,
      * using an atomic write operation if the implementation supports it.
-     *
+     * 
                      
      * This method assumes the container does not already contain a blob of the same blobName.  If a blob by the
      * same name already exists, the operation will fail and an {@link IOException} will be thrown.
      *
diff --git a/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java
index 394855671688a..b6644ffd16bab 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/fs/FsBlobContainer.java
@@ -69,7 +69,7 @@
 /**
  * A file system based implementation of {@link org.opensearch.common.blobstore.BlobContainer}.
  * All blobs in the container are stored on a file system, the location of which is specified by the {@link BlobPath}.
- *
+ * 
                      
  * Note that the methods in this implementation of {@link org.opensearch.common.blobstore.BlobContainer} may
  * additionally throw a {@link java.lang.SecurityException} if the configured {@link java.lang.SecurityManager}
  * does not permit read and/or write access to the underlying files.
@@ -258,7 +258,7 @@ public static String tempBlobName(final String blobName) {
 
     /**
      * Returns true if the blob is a leftover temporary blob.
-     *
+     * 
                      
      * The temporary blobs might be left after failed atomic write operation.
      */
     public static boolean isTempBlobName(final String blobName) {
diff --git a/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java b/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java
index de4e6ad433c55..c9b498c3ec6fa 100644
--- a/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java
+++ b/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java
@@ -204,7 +204,7 @@ private long limit(long bytes, String label, double overheadConstant, long memor
     /**
      * Add an exact number of bytes, not checking for tripping the
      * circuit breaker. This bypasses the overheadConstant multiplication.
-     *
+     * 
                      
      * Also does not check with the parent breaker to see if the parent limit
      * has been exceeded.
      *
diff --git a/server/src/main/java/org/opensearch/common/cache/Cache.java b/server/src/main/java/org/opensearch/common/cache/Cache.java
index 0ebef1556424b..0b2b608b55df0 100644
--- a/server/src/main/java/org/opensearch/common/cache/Cache.java
+++ b/server/src/main/java/org/opensearch/common/cache/Cache.java
@@ -403,7 +403,7 @@ private V get(K key, long now, Consumer> onExpiration) {
      * If the specified key is not already associated with a value (or is mapped to null), attempts to compute its
      * value using the given mapping function and enters it into this map unless null. The load method for a given key
      * will be invoked at most once.
-     *
+     * 
                      
      * Use of different {@link CacheLoader} implementations on the same key concurrently may result in only the first
      * loader function being called and the second will be returned the result provided by the first including any exceptions
      * thrown during the execution of the first.
diff --git a/server/src/main/java/org/opensearch/common/collect/CopyOnWriteHashMap.java b/server/src/main/java/org/opensearch/common/collect/CopyOnWriteHashMap.java
index 5ce77cdc75fe5..de4304f0e1fba 100644
--- a/server/src/main/java/org/opensearch/common/collect/CopyOnWriteHashMap.java
+++ b/server/src/main/java/org/opensearch/common/collect/CopyOnWriteHashMap.java
@@ -49,15 +49,15 @@
 
 /**
  * An immutable map whose writes result in a new copy of the map to be created.
- *
+ * 
                      
  * This is essentially a hash array mapped trie: inner nodes use a bitmap in
  * order to map hashes to slots by counting ones. In case of a collision (two
  * values having the same 32-bits hash), a leaf node is created which stores
  * and searches for values sequentially.
- *
+ * 
                      
  * Reads and writes both perform in logarithmic time. Null keys and values are
  * not supported.
- *
+ * 
                      
  * This structure might need to perform several object creations per write so
  * it is better suited for work-loads that are not too write-intensive.
  *
@@ -250,7 +250,7 @@ public static  T[] insertElement(final T[] array, final T element, final int
      * and use a bitmap in order to associate hashes to them. For example, if
      * an inner node contains 5 values, then 5 bits will be set in the bitmap
      * and the ordinal of the bit set in this bit map will be the slot number.
-     *
+     * 
                      
      * As a consequence, the number of slots in an inner node is equal to the
      * number of one bits in the bitmap.
      *
diff --git a/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java b/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java
index 0334d367ffdbc..1622457ba27cc 100644
--- a/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java
+++ b/server/src/main/java/org/opensearch/common/geo/GeoShapeType.java
@@ -221,11 +221,11 @@ void validateLinearRing(CoordinateNode coordinates, boolean coerce) {
 
         @Override
         CoordinateNode validate(CoordinateNode coordinates, boolean coerce) {
-            /**
-             * Per GeoJSON spec (http://geojson.org/geojson-spec.html#linestring)
-             * A LinearRing is closed LineString with 4 or more positions. The first and last positions
-             * are equivalent (they represent equivalent points). Though a LinearRing is not explicitly
-             * represented as a GeoJSON geometry type, it is referred to in the Polygon geometry type definition.
+            /*
+              Per GeoJSON spec (http://geojson.org/geojson-spec.html#linestring)
+              A LinearRing is closed LineString with 4 or more positions. The first and last positions
+              are equivalent (they represent equivalent points). Though a LinearRing is not explicitly
+              represented as a GeoJSON geometry type, it is referred to in the Polygon geometry type definition.
              */
             if (coordinates.children == null || coordinates.children.isEmpty()) {
                 throw new OpenSearchParseException(
diff --git a/server/src/main/java/org/opensearch/common/geo/GeoUtils.java b/server/src/main/java/org/opensearch/common/geo/GeoUtils.java
index 393c238cb3b2f..8c566c4191e4f 100644
--- a/server/src/main/java/org/opensearch/common/geo/GeoUtils.java
+++ b/server/src/main/java/org/opensearch/common/geo/GeoUtils.java
@@ -665,7 +665,7 @@ public static GeoPoint parseFromString(String val) {
 
     /**
      * Parse a precision that can be expressed as an integer or a distance measure like "1km", "10m".
-     *
+     * 
                      
      * The precision is expressed as a number between 1 and 12 and indicates the length of geohash
      * used to represent geo points.
      *
@@ -696,7 +696,7 @@ public static int parsePrecision(XContentParser parser) throws IOException, Open
 
     /**
      * Checks that the precision is within range supported by opensearch - between 1 and 12
-     *
+     * 
                      
      * Returns the precision value if it is in the range and throws an IllegalArgumentException if it
      * is outside the range.
      */
diff --git a/server/src/main/java/org/opensearch/common/geo/GeometryFormat.java b/server/src/main/java/org/opensearch/common/geo/GeometryFormat.java
index 56146fc8197be..93c7f4b93679a 100644
--- a/server/src/main/java/org/opensearch/common/geo/GeometryFormat.java
+++ b/server/src/main/java/org/opensearch/common/geo/GeometryFormat.java
@@ -63,7 +63,7 @@ public interface GeometryFormat {
 
     /**
      * Serializes the geometry into a standard Java object.
-     *
+     * 
                      
      * For example, the GeoJson format returns the geometry as a map, while WKT returns a string.
      */
     Object toXContentAsObject(ParsedFormat geometry);
diff --git a/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java b/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java
index b436787220eb0..9e118ab2de3a5 100644
--- a/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java
+++ b/server/src/main/java/org/opensearch/common/geo/builders/PolygonBuilder.java
@@ -177,11 +177,11 @@ public PolygonBuilder close() {
     }
 
     private static void validateLinearRing(LineStringBuilder lineString) {
-        /**
-         * Per GeoJSON spec (http://geojson.org/geojson-spec.html#linestring)
-         * A LinearRing is closed LineString with 4 or more positions. The first and last positions
-         * are equivalent (they represent equivalent points). Though a LinearRing is not explicitly
-         * represented as a GeoJSON geometry type, it is referred to in the Polygon geometry type definition.
+        /*
+          Per GeoJSON spec (http://geojson.org/geojson-spec.html#linestring)
+          A LinearRing is closed LineString with 4 or more positions. The first and last positions
+          are equivalent (they represent equivalent points). Though a LinearRing is not explicitly
+          represented as a GeoJSON geometry type, it is referred to in the Polygon geometry type definition.
          */
         List points = lineString.coordinates;
         if (points.size() < 4) {
diff --git a/server/src/main/java/org/opensearch/common/geo/parsers/GeoJsonParser.java b/server/src/main/java/org/opensearch/common/geo/parsers/GeoJsonParser.java
index 37d42ce600b6d..8d473ae6721d2 100644
--- a/server/src/main/java/org/opensearch/common/geo/parsers/GeoJsonParser.java
+++ b/server/src/main/java/org/opensearch/common/geo/parsers/GeoJsonParser.java
@@ -52,7 +52,7 @@
 
 /**
  * Parses shape geometry represented in geojson
- *
+ * 
                      
  * complies with geojson specification: https://tools.ietf.org/html/rfc7946
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/common/geo/parsers/GeoWKTParser.java b/server/src/main/java/org/opensearch/common/geo/parsers/GeoWKTParser.java
index d99d1daf46a2a..b199da0f3691a 100644
--- a/server/src/main/java/org/opensearch/common/geo/parsers/GeoWKTParser.java
+++ b/server/src/main/java/org/opensearch/common/geo/parsers/GeoWKTParser.java
@@ -58,7 +58,7 @@
 
 /**
  * Parses shape geometry represented in WKT format
- *
+ * 
                      
  * complies with OGC® document: 12-063r5 and ISO/IEC 13249-3:2016 standard
  * located at http://docs.opengeospatial.org/is/12-063r5/12-063r5.html
  *
diff --git a/server/src/main/java/org/opensearch/common/hash/MurmurHash3.java b/server/src/main/java/org/opensearch/common/hash/MurmurHash3.java
index 8ba0bd7ee1be4..e481ffd460798 100644
--- a/server/src/main/java/org/opensearch/common/hash/MurmurHash3.java
+++ b/server/src/main/java/org/opensearch/common/hash/MurmurHash3.java
@@ -93,7 +93,7 @@ protected static long fmix(long k) {
 
     /**
      * Compute the hash of the MurmurHash3_x64_128 hashing function.
-     *
+     * 
                      
      * Note, this hashing function might be used to persist hashes, so if the way hashes are computed
      * changes for some reason, it needs to be addressed (like in BloomFilter and MurmurHashField).
      */
diff --git a/server/src/main/java/org/opensearch/common/inject/Initializer.java b/server/src/main/java/org/opensearch/common/inject/Initializer.java
index e806eba6df707..b88b01c03c018 100644
--- a/server/src/main/java/org/opensearch/common/inject/Initializer.java
+++ b/server/src/main/java/org/opensearch/common/inject/Initializer.java
@@ -68,9 +68,8 @@ class Initializer {
     /**
      * Registers an instance for member injection when that step is performed.
      *
-     * @param instance an instance that optionally has members to be injected (each annotated with
-     * @param source   the source location that this injection was requested
-     * @Inject).
+     * @param instance an instance that optionally has members to be injected (each annotated with {@code @Inject}).
+     * @param source the source location that this injection was requested
      */
     public  Initializable requestInjection(InjectorImpl injector, T instance, Object source, Set injectionPoints) {
         Objects.requireNonNull(source);
diff --git a/server/src/main/java/org/opensearch/common/io/stream/Streamables.java b/server/src/main/java/org/opensearch/common/io/stream/Streamables.java
index fd9ed6e5b18fe..f1e5f5f22d527 100644
--- a/server/src/main/java/org/opensearch/common/io/stream/Streamables.java
+++ b/server/src/main/java/org/opensearch/common/io/stream/Streamables.java
@@ -40,7 +40,7 @@ public static void registerStreamables() {
      * Registers writers by class type
      */
     private static void registerWriters() {
-        /** {@link GeoPoint} */
+        /* {@link GeoPoint} */
         WriteableRegistry.registerWriter(GeoPoint.class, (o, v) -> {
             o.writeByte((byte) 22);
             ((GeoPoint) v).writeTo(o);
@@ -53,7 +53,7 @@ private static void registerWriters() {
      * NOTE: see {@code StreamOutput#WRITERS} for all registered ordinals
      */
     private static void registerReaders() {
-        /** {@link GeoPoint} */
+        /* {@link GeoPoint} */
         WriteableRegistry.registerReader(Byte.valueOf((byte) 22), GeoPoint::new);
     }
 }
diff --git a/server/src/main/java/org/opensearch/common/joda/JodaDateMathParser.java b/server/src/main/java/org/opensearch/common/joda/JodaDateMathParser.java
index 0de6dec1c25bd..ae38e9a6a8073 100644
--- a/server/src/main/java/org/opensearch/common/joda/JodaDateMathParser.java
+++ b/server/src/main/java/org/opensearch/common/joda/JodaDateMathParser.java
@@ -46,7 +46,7 @@
 
 /**
  * A parser for date/time formatted text with optional date math.
- *
+ * 
                      
  * The format of the datetime is configurable, and unix timestamps can also be used. Datemath
  * is appended to a datetime with the following syntax:
  * ||[+-/](\d+)?[yMwdhHms].
diff --git a/server/src/main/java/org/opensearch/common/logging/JsonThrowablePatternConverter.java b/server/src/main/java/org/opensearch/common/logging/JsonThrowablePatternConverter.java
index e259d5d9e3e33..ed324e4e62d8f 100644
--- a/server/src/main/java/org/opensearch/common/logging/JsonThrowablePatternConverter.java
+++ b/server/src/main/java/org/opensearch/common/logging/JsonThrowablePatternConverter.java
@@ -47,7 +47,7 @@
 
  * Outputs the Throwable portion of the LoggingEvent as a Json formatted field with array
  * "exception": [ "stacktrace", "lines", "as", "array", "elements" ]
- *
+ * 
                      
  * Reusing @link org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter which already converts a Throwable from
  * LoggingEvent into a multiline string
  *
diff --git a/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java b/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java
index 0ffd633e5a967..17b75ab22f3ed 100644
--- a/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java
+++ b/server/src/main/java/org/opensearch/common/lucene/ShardCoreKeyMap.java
@@ -55,7 +55,7 @@
  * mappings as segments that were not known before are added and prevents the
  * structure from growing indefinitely by registering close listeners on these
  * segments so that at any time it only tracks live segments.
- *
+ * 
                      
  * NOTE: This is heavy. Avoid using this class unless absolutely required.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/common/rounding/Rounding.java b/server/src/main/java/org/opensearch/common/rounding/Rounding.java
index 857031bc783f4..41e808b64f7d9 100644
--- a/server/src/main/java/org/opensearch/common/rounding/Rounding.java
+++ b/server/src/main/java/org/opensearch/common/rounding/Rounding.java
@@ -45,7 +45,7 @@
 
 /**
  * A strategy for rounding long values.
- *
+ * 
                      
  * Use the java based Rounding class where applicable
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java
index 8b7a2a82e5cb1..117ed66fcb451 100644
--- a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java
@@ -254,7 +254,7 @@ public synchronized  void addSettingsUpdateConsumer(Setting setting, Consu
     /**
      * Adds a settings consumer that is only executed if any setting in the supplied list of settings is changed. In that case all the
      * settings are specified in the argument are returned.
-     *
+     * 
                      
      * Also automatically adds empty consumers for all settings in order to activate logging
      */
     public synchronized void addSettingsUpdateConsumer(Consumer consumer, List> settings) {
@@ -265,7 +265,7 @@ public synchronized void addSettingsUpdateConsumer(Consumer consumer,
      * Adds a settings consumer that is only executed if any setting in the supplied list of settings is changed. In that case all the
      * settings are specified in the argument are returned.  The validator is run across all specified settings before the settings are
      * applied.
-     *
+     * 
                      
      * Also automatically adds empty consumers for all settings in order to activate logging
      */
     public synchronized void addSettingsUpdateConsumer(
diff --git a/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java
index f25dd872fc703..1ad3b7ab8875a 100644
--- a/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java
@@ -88,7 +88,7 @@
 
 /**
  * A disk based container for sensitive settings in OpenSearch.
- *
+ * 
                      
  * Loading a keystore has 2 phases. First, call {@link #load(Path)}. Then call
  * {@link #decrypt(char[])} with the keystore password, or an empty char array if
  * {@link #hasPassword()} is {@code false}.  Loading and decrypting should happen
@@ -147,7 +147,7 @@ private static class Entry {
 
     /**
      * The number of bits for the cipher key.
-     *
+     * 
                      
      * Note: The Oracle JDK 8 ships with a limited JCE policy that restricts key length for AES to 128 bits.
      * This can be increased to 256 bits once minimum java 9 is the minimum java version.
      * See http://www.oracle.com/technetwork/java/javase/terms/readme/jdk9-readme-3852447.html#jce
@@ -234,7 +234,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException {
 
     /**
      * Loads information about the OpenSearch keystore from the provided config directory.
-     *
+     * 
                      
      * {@link #decrypt(char[])} must be called before reading or writing any entries.
      * Returns {@code null} if no keystore exists.
      */
@@ -358,7 +358,7 @@ private Cipher createCipher(int opmode, char[] password, byte[] salt, byte[] iv)
 
     /**
      * Decrypts the underlying keystore data.
-     *
+     * 
                      
      * This may only be called once.
      */
     public void decrypt(char[] password) throws GeneralSecurityException, IOException {
diff --git a/server/src/main/java/org/opensearch/common/settings/SecureSetting.java b/server/src/main/java/org/opensearch/common/settings/SecureSetting.java
index f2ccc01a4c7e6..1855270b016b3 100644
--- a/server/src/main/java/org/opensearch/common/settings/SecureSetting.java
+++ b/server/src/main/java/org/opensearch/common/settings/SecureSetting.java
@@ -45,7 +45,7 @@
 
 /**
  * A secure setting.
- *
+ * 
                      
  * This class allows access to settings from the OpenSearch keystore.
  *
  * @opensearch.internal
@@ -152,7 +152,7 @@ public void diff(Settings.Builder builder, Settings source, Settings defaultSett
 
     /**
      * A setting which contains a sensitive string.
-     *
+     * 
                      
      * This may be any sensitive string, e.g. a username, a password, an auth token, etc.
      */
     public static Setting secureString(String name, Setting fallback, Property... properties) {
@@ -179,7 +179,7 @@ public static Setting insecureString(String name, String secureNam
 
     /**
      * A setting which contains a file. Reading the setting opens an input stream to the file.
-     *
+     * 
                      
      * This may be any sensitive file, e.g. a set of credentials normally in plaintext.
      */
     public static Setting secureFile(String name, Setting fallback, Property... properties) {
diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java
index d202feb0786bf..0557884f0f8ad 100644
--- a/server/src/main/java/org/opensearch/common/settings/Settings.java
+++ b/server/src/main/java/org/opensearch/common/settings/Settings.java
@@ -1222,7 +1222,7 @@ public boolean shouldRemoveMissingPlaceholder(String placeholderName) {
 
         /**
          * Checks that all settings(except archived settings and wildcards) in the builder start with the specified prefix.
-         *
+         * 
                      
          * If a setting doesn't start with the prefix, the builder appends the prefix to such setting.
          */
         public Builder normalizePrefix(String prefix) {
diff --git a/server/src/main/java/org/opensearch/common/time/DateFormatters.java b/server/src/main/java/org/opensearch/common/time/DateFormatters.java
index 6c8b9282d8a77..e74ab687b903b 100644
--- a/server/src/main/java/org/opensearch/common/time/DateFormatters.java
+++ b/server/src/main/java/org/opensearch/common/time/DateFormatters.java
@@ -2172,10 +2172,10 @@ static DateFormatter forPattern(String input) {
      * or Instant.from(accessor). This results in a huge performance penalty and should be prevented
      * This method prevents exceptions by querying the accessor for certain capabilities
      * and then act on it accordingly
-     *
+     * 
                      
      * This action assumes that we can reliably fall back to some defaults if not all parts of a
      * zoned date time are set
-     *
+     * 
                      
      * - If a zoned date time is passed, it is returned
      * - If no timezone is found, ZoneOffset.UTC is used
      * - If we find a time and a date, converting to a ZonedDateTime is straight forward,
diff --git a/server/src/main/java/org/opensearch/common/time/DateMathParser.java b/server/src/main/java/org/opensearch/common/time/DateMathParser.java
index f6573eaa90286..7088d6cb7a498 100644
--- a/server/src/main/java/org/opensearch/common/time/DateMathParser.java
+++ b/server/src/main/java/org/opensearch/common/time/DateMathParser.java
@@ -64,12 +64,12 @@ default Instant parse(String text, LongSupplier now, boolean roundUpProperty, Da
 
     /**
      * Parse text, that potentially contains date math into the milliseconds since the epoch
-     *
+     * 
                      
      * Examples are
-     *
+     * 
                      
      * 2014-11-18||-2y subtracts two years from the input date
      * now/m           rounds the current time to minute granularity
-     *
+     * 
                      
      * Supported rounding units are
      * y    year
      * M    month
diff --git a/server/src/main/java/org/opensearch/common/time/DateUtils.java b/server/src/main/java/org/opensearch/common/time/DateUtils.java
index 021b8a3be8b23..7ab395a1117e7 100644
--- a/server/src/main/java/org/opensearch/common/time/DateUtils.java
+++ b/server/src/main/java/org/opensearch/common/time/DateUtils.java
@@ -342,7 +342,7 @@ public static long toMilliSeconds(long nanoSecondsSinceEpoch) {
 
     /**
      * Rounds the given utc milliseconds sicne the epoch down to the next unit millis
-     *
+     * 
                      
      * Note: This does not check for correctness of the result, as this only works with units smaller or equal than a day
      *       In order to ensure the performance of this methods, there are no guards or checks in it
      *
diff --git a/server/src/main/java/org/opensearch/common/time/DateUtilsRounding.java b/server/src/main/java/org/opensearch/common/time/DateUtilsRounding.java
index f3459a5857b9e..7fc39e063efb5 100644
--- a/server/src/main/java/org/opensearch/common/time/DateUtilsRounding.java
+++ b/server/src/main/java/org/opensearch/common/time/DateUtilsRounding.java
@@ -32,12 +32,12 @@
  * This class has been copied from different locations within the joda time package, as
  * these methods fast when used for rounding, as they do not require conversion to java
  * time objects
- *
+ * 
                      
  * This code has been copied from jodatime 2.10.1
  * The source can be found at https://github.com/JodaOrg/joda-time/tree/v2.10.1
- *
+ * 
                      
  * See following methods have been copied (along with required helper variables)
- *
+ * 
                      
  * - org.joda.time.chrono.GregorianChronology.calculateFirstDayOfYearMillis(int year)
  * - org.joda.time.chrono.BasicChronology.getYear(int year)
  * - org.joda.time.chrono.BasicGJChronology.getMonthOfYear(long utcMillis, int year)
diff --git a/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java b/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java
index 14191357fb8aa..f711b14aeb928 100644
--- a/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java
+++ b/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java
@@ -275,7 +275,7 @@ public TemporalAccessor parse(String input) {
      * it will continue iterating if the previous parser failed. The pattern must fully match, meaning whole input was used.
      * This also means that this method depends on DateTimeFormatter.ClassicFormat.parseObject
      * which does not throw exceptions when parsing failed.
-     *
+     * 
                      
      * The approach with collection of parsers was taken because java-time requires ordering on optional (composite)
      * patterns. Joda does not suffer from this.
      * https://bugs.openjdk.java.net/browse/JDK-8188771
diff --git a/server/src/main/java/org/opensearch/common/time/JavaDateMathParser.java b/server/src/main/java/org/opensearch/common/time/JavaDateMathParser.java
index 0536324b6516b..340901e7ac8e2 100644
--- a/server/src/main/java/org/opensearch/common/time/JavaDateMathParser.java
+++ b/server/src/main/java/org/opensearch/common/time/JavaDateMathParser.java
@@ -51,7 +51,7 @@
 
 /**
  * A parser for date/time formatted text with optional date math.
- *
+ * 
                      
  * The format of the datetime is configurable, and unix timestamps can also be used. Datemath
  * is appended to a datetime with the following syntax:
  * ||[+-/](\d+)?[yMwdhHms].
diff --git a/server/src/main/java/org/opensearch/common/util/BinarySearcher.java b/server/src/main/java/org/opensearch/common/util/BinarySearcher.java
index ca63c170c0ccd..e4315f8699206 100644
--- a/server/src/main/java/org/opensearch/common/util/BinarySearcher.java
+++ b/server/src/main/java/org/opensearch/common/util/BinarySearcher.java
@@ -34,14 +34,14 @@
 
 /**
  * Performs binary search on an arbitrary data structure.
- *
+ * 
                      
  * To do a search, create a subclass and implement custom {@link #compare(int)} and {@link #distance(int)} methods.
- *
+ * 
                      
  * {@link BinarySearcher} knows nothing about the value being searched for or the underlying data structure.
  * These things should be determined by the subclass in its overridden methods.
- *
+ * 
                      
  * Refer to {@link BigArrays.DoubleBinarySearcher} for an example.
- *
+ * 
                      
  * NOTE: this class is not thread safe
  *
  * @opensearch.internal
@@ -74,7 +74,7 @@ private int getClosestIndex(int index1, int index2) {
     /**
      * Uses a binary search to determine the index of the element within the index range {from, ... , to} that is
      * closest to the search value.
-     *
+     * 
                      
      * Unlike most binary search implementations, the value being searched for is not an argument to search method.
      * Rather, this value should be stored by the subclass along with the underlying array.
      *
diff --git a/server/src/main/java/org/opensearch/common/util/CancellableThreads.java b/server/src/main/java/org/opensearch/common/util/CancellableThreads.java
index 8bc3ca3affb12..67dd4b848f4c0 100644
--- a/server/src/main/java/org/opensearch/common/util/CancellableThreads.java
+++ b/server/src/main/java/org/opensearch/common/util/CancellableThreads.java
@@ -45,7 +45,7 @@
  * A utility class for multi threaded operation that needs to be cancellable via interrupts. Every cancellable operation should be
  * executed via {@link #execute(Interruptible)}, which will capture the executing thread and make sure it is interrupted in the case
  * of cancellation.
- *
+ * 
                      
  * Cancellation policy: This class does not support external interruption via Thread#interrupt(). Always use #cancel() instead.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/common/util/CuckooFilter.java b/server/src/main/java/org/opensearch/common/util/CuckooFilter.java
index 0c792b37ccfa9..28b55f70855d6 100644
--- a/server/src/main/java/org/opensearch/common/util/CuckooFilter.java
+++ b/server/src/main/java/org/opensearch/common/util/CuckooFilter.java
@@ -46,33 +46,33 @@
 
 /**
  * An approximate set membership datastructure
- *
+ * 
                      
  * CuckooFilters are similar to Bloom Filters in usage; values are inserted, and the Cuckoo
  * can be asked if it has seen a particular value before.  Because the structure is approximate,
  * it can return false positives (says it has seen an item when it has not).  False negatives
  * are not possible though; if the structure says it _has not_ seen an item, that can be
  * trusted.
- *
+ * 
                      
  * The filter can "saturate" at which point the map has hit it's configured load factor (or near enough
  * that a large number of evictions are not able to find a free slot) and will refuse to accept
  * any new insertions.
- *
+ * 
                      
  * NOTE: this version does not support deletions, and as such does not save duplicate
  * fingerprints (e.g. when inserting, if the fingerprint is already present in the
  * candidate buckets, it is not inserted).  By not saving duplicates, the CuckooFilter
  * loses the ability to delete values.  But not by allowing deletions, we can save space
  * (do not need to waste slots on duplicate fingerprints), and we do not need to worry
  * about inserts "overflowing" a bucket because the same item has been repeated repeatedly
- *
+ * 
                      
  * NOTE: this CuckooFilter exposes a number of Expert APIs which assume the caller has
  * intimate knowledge about how the algorithm works.  It is recommended to use
  * {@link SetBackedScalingCuckooFilter} instead.
- *
+ * 
                      
  * Based on the paper:
- *
+ * 
                      
  * Fan, Bin, et al. "Cuckoo filter: Practically better than bloom."
  * Proceedings of the 10th ACM International on Conference on emerging Networking Experiments and Technologies. ACM, 2014.
- *
+ * 
                      
  * https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf
  *
  * @opensearch.internal
@@ -200,7 +200,7 @@ public int getCount() {
     /**
      * Returns the number of buckets that has been chosen based
      * on the initial configuration
-     *
+     * 
                      
      * Expert-level API
      */
     int getNumBuckets() {
@@ -209,7 +209,7 @@ int getNumBuckets() {
 
     /**
      * Returns the number of bits used per entry
-     *
+     * 
                      
      * Expert-level API
      */
     int getBitsPerEntry() {
@@ -220,7 +220,7 @@ int getBitsPerEntry() {
      * Returns the cached fingerprint mask.  This is simply a mask for the
      * first bitsPerEntry bits, used by {@link CuckooFilter#fingerprint(int, int, int)}
      * to generate the fingerprint of a hash
-     *
+     * 
                      
      * Expert-level API
      */
     int getFingerprintMask() {
@@ -230,7 +230,7 @@ int getFingerprintMask() {
     /**
      * Returns an iterator that returns the long[] representation of each bucket.  The value
      * inside each long will be a fingerprint (or 0L, representing empty).
-     *
+     * 
                      
      * Expert-level API
      */
     Iterator getBuckets() {
@@ -267,7 +267,7 @@ boolean mightContain(long hash) {
 
     /**
      * Returns true if the bucket or it's alternate bucket contains the fingerprint.
-     *
+     * 
                      
      * Expert-level API, use {@link CuckooFilter#mightContain(long)} to check if
      * a value is in the filter.
      */
@@ -307,7 +307,7 @@ boolean add(long hash) {
     /**
      * Attempts to merge the fingerprint into the specified bucket or it's alternate bucket.
      * Returns true if the insertion was successful, false if the filter is saturated.
-     *
+     * 
                      
      * Expert-level API, use {@link CuckooFilter#add(long)} to insert
      * values into the filter
      */
@@ -351,7 +351,7 @@ boolean mergeFingerprint(int bucket, int fingerprint) {
      * Low-level insert method. Attempts to write the fingerprint into an empty entry
      * at this bucket's position.  Returns true if that was sucessful, false if all entries
      * were occupied.
-     *
+     * 
                      
      * If the fingerprint already exists in one of the entries, it will not duplicate the
      * fingerprint like the original paper.  This means the filter _cannot_ support deletes,
      * but is not sensitive to "overflowing" buckets with repeated inserts
@@ -376,10 +376,10 @@ private boolean tryInsert(int bucket, int fingerprint) {
 
     /**
      * Converts a hash into a bucket index (primary or alternate).
-     *
+     * 
                      
      * If the hash is negative, this flips the bits.  The hash is then modulo numBuckets
      * to get the final index.
-     *
+     * 
                      
      * Expert-level API
      */
     static int hashToIndex(int hash, int numBuckets) {
@@ -388,16 +388,16 @@ static int hashToIndex(int hash, int numBuckets) {
 
     /**
      * Calculates the alternate bucket for a given bucket:fingerprint tuple
-     *
+     * 
                      
      * The alternate bucket is the fingerprint multiplied by a mixing constant,
      * then xor'd against the bucket.  This new value is modulo'd against
      * the buckets via {@link CuckooFilter#hashToIndex(int, int)} to get the final
      * index.
-     *
+     * 
                      
      * Note that the xor makes this operation reversible as long as we have the
      * fingerprint and current bucket (regardless of if that bucket was the primary
      * or alternate).
-     *
+     * 
                      
      * Expert-level API
      */
     static int alternateIndex(int bucket, int fingerprint, int numBuckets) {
@@ -424,10 +424,10 @@ private int getOffset(int bucket, int position) {
 
     /**
      * Calculates the fingerprint for a given hash.
-     *
+     * 
                      
      * The fingerprint is simply the first `bitsPerEntry` number of bits that are non-zero.
      * If the entire hash is zero, `(int) 1` is used
-     *
+     * 
                      
      * Expert-level API
      */
     static int fingerprint(int hash, int bitsPerEntry, int fingerprintMask) {
@@ -501,7 +501,7 @@ private double getLoadFactor(int b) {
      * Calculates the optimal number of buckets for this filter.  The xor used in the bucketing
      * algorithm requires this to be a power of two, so the optimal number of buckets will
      * be rounded to the next largest power of two where applicable.
-     *
+     * 
                      
      * TODO: there are schemes to avoid powers of two, might want to investigate those
      */
     private int getNumBuckets(long capacity, double loadFactor, int b) {
diff --git a/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java b/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java
index e99eb751babe8..a635160844159 100644
--- a/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java
+++ b/server/src/main/java/org/opensearch/common/util/SetBackedScalingCuckooFilter.java
@@ -75,7 +75,7 @@ public class SetBackedScalingCuckooFilter implements Writeable {
      * This set is used to track the insertions before we convert over to an approximate
      * filter. This gives us 100% accuracy for small cardinalities.  This will be null
      * if isSetMode = false;
-     *
+     * 
                      
      * package-private for testing
      */
     Set hashes;
@@ -178,7 +178,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     /**
      * Registers a circuit breaker with the datastructure.
-     *
+     * 
                      
      * CuckooFilter's can "saturate" and refuse to accept any new values.  When this happens,
      * the datastructure scales by adding a new filter.  This new filter's bytes will be tracked
      * in the registered breaker when configured.
diff --git a/server/src/main/java/org/opensearch/common/util/TokenBucket.java b/server/src/main/java/org/opensearch/common/util/TokenBucket.java
index d2e7e836bf07f..a9ebb86eed8a2 100644
--- a/server/src/main/java/org/opensearch/common/util/TokenBucket.java
+++ b/server/src/main/java/org/opensearch/common/util/TokenBucket.java
@@ -20,7 +20,7 @@
 public class TokenBucket {
     /**
      * Defines a monotonically increasing counter.
-     *
+     * 
                      
      * Usage examples:
      * 1. clock = System::nanoTime can be used to perform rate-limiting per unit time
      * 2. clock = AtomicLong::get can be used to perform rate-limiting per unit number of operations
diff --git a/server/src/main/java/org/opensearch/common/util/URIPattern.java b/server/src/main/java/org/opensearch/common/util/URIPattern.java
index a3c385e5ea660..49e4b53e20740 100644
--- a/server/src/main/java/org/opensearch/common/util/URIPattern.java
+++ b/server/src/main/java/org/opensearch/common/util/URIPattern.java
@@ -39,9 +39,9 @@
 
 /**
  * URI Pattern matcher
- *
+ * 
                      
  * The pattern is URI in which authority, path, query and fragment can be replace with simple pattern.
- *
+ * 
                      
  * For example: foobar://*.local/some_path/*?*#* will match all uris with schema foobar in local domain
  * with any port, with path that starts some_path and with any query and fragment.
  *
diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ListenableFuture.java b/server/src/main/java/org/opensearch/common/util/concurrent/ListenableFuture.java
index b1f4714a90e8e..4357254176358 100644
--- a/server/src/main/java/org/opensearch/common/util/concurrent/ListenableFuture.java
+++ b/server/src/main/java/org/opensearch/common/util/concurrent/ListenableFuture.java
@@ -73,7 +73,7 @@ public void addListener(ActionListener listener, ExecutorService executor) {
      * notified of a response or exception in a runnable submitted to the ExecutorService provided.
      * If the future has completed, the listener will be notified immediately without forking to
      * a different thread.
-     *
+     * 
                      
      * It will apply the provided ThreadContext (if not null) when executing the listening.
      */
     public void addListener(ActionListener listener, ExecutorService executor, ThreadContext threadContext) {
diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/PrioritizedOpenSearchThreadPoolExecutor.java b/server/src/main/java/org/opensearch/common/util/concurrent/PrioritizedOpenSearchThreadPoolExecutor.java
index b4673d9534922..95df4486b9d7b 100644
--- a/server/src/main/java/org/opensearch/common/util/concurrent/PrioritizedOpenSearchThreadPoolExecutor.java
+++ b/server/src/main/java/org/opensearch/common/util/concurrent/PrioritizedOpenSearchThreadPoolExecutor.java
@@ -118,8 +118,9 @@ private void addPending(List runnables, List pending, boolean
                 TieBreakingPrioritizedRunnable t = (TieBreakingPrioritizedRunnable) runnable;
                 Runnable innerRunnable = t.runnable;
                 if (innerRunnable != null) {
-                    /** innerRunnable can be null if task is finished but not removed from executor yet,
-                     * see {@link TieBreakingPrioritizedRunnable#run} and {@link TieBreakingPrioritizedRunnable#runAndClean}
+                    /*
+                      innerRunnable can be null if task is finished but not removed from executor yet,
+                      see {@link TieBreakingPrioritizedRunnable#run} and {@link TieBreakingPrioritizedRunnable#runAndClean}
                      */
                     pending.add(new Pending(super.unwrap(innerRunnable), t.priority(), t.insertionOrder, executing));
                 }
diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java
index 4888d25e4a640..fc2e4217bae79 100644
--- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java
@@ -143,10 +143,10 @@ public void unregisterThreadContextStatePropagator(final ThreadContextStatePropa
      */
     public StoredContext stashContext() {
         final ThreadContextStruct context = threadLocal.get();
-        /**
-         * X-Opaque-ID should be preserved in a threadContext in order to propagate this across threads.
-         * This is needed so the DeprecationLogger in another thread can see the value of X-Opaque-ID provided by a user.
-         * Otherwise when context is stash, it should be empty.
+        /*
+          X-Opaque-ID should be preserved in a threadContext in order to propagate this across threads.
+          This is needed so the DeprecationLogger in another thread can see the value of X-Opaque-ID provided by a user.
+          Otherwise when context is stash, it should be empty.
          */
 
         ThreadContextStruct threadContextStruct = DEFAULT_CONTEXT.putPersistent(context.persistentHeaders);
diff --git a/server/src/main/java/org/opensearch/common/xcontent/LoggingDeprecationHandler.java b/server/src/main/java/org/opensearch/common/xcontent/LoggingDeprecationHandler.java
index 06d139fa93195..05fc968737394 100644
--- a/server/src/main/java/org/opensearch/common/xcontent/LoggingDeprecationHandler.java
+++ b/server/src/main/java/org/opensearch/common/xcontent/LoggingDeprecationHandler.java
@@ -54,7 +54,7 @@ public class LoggingDeprecationHandler implements DeprecationHandler {
     public static final LoggingDeprecationHandler INSTANCE = new LoggingDeprecationHandler();
     /**
      * The logger to which to send deprecation messages.
-     *
+     * 
                      
      * This uses ParseField's logger because that is the logger that
      * we have been using for many releases for deprecated fields.
      * Changing that will require some research to make super duper
diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java
index 798a58551457f..17bb0a1de267b 100644
--- a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java
+++ b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java
@@ -494,7 +494,7 @@ public static BytesReference toXContent(ToXContent toXContent, XContentType xCon
 
     /**
      * Returns the contents of an object as an unparsed BytesReference
-     *
+     * 
                      
      * This is useful for things like mappings where we're copying bytes around but don't
      * actually need to parse their contents, and so avoids building large maps of maps
      * unnecessarily
diff --git a/server/src/main/java/org/opensearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/opensearch/common/xcontent/support/XContentMapValues.java
index adfa871cbfcbe..a87edbb949d39 100644
--- a/server/src/main/java/org/opensearch/common/xcontent/support/XContentMapValues.java
+++ b/server/src/main/java/org/opensearch/common/xcontent/support/XContentMapValues.java
@@ -117,12 +117,11 @@ private static void extractRawValues(List values, List part, String[] pa
 
     /**
      * For the provided path, return its value in the xContent map.
-     *
+     * 
                      
      * Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values
      * can be returned.
      *
      * @param path the value's path in the map.
-     *
      * @return the value associated with the path in the map or 'null' if the path does not exist.
      */
     public static Object extractValue(String path, Map map) {
@@ -138,7 +137,7 @@ public static Object extractValue(Map map, String... pathElements) {
 
     /**
      * For the provided path, return its value in the xContent map.
-     *
+     * 
                      
      * Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values
      * can be returned.
      *
@@ -197,13 +196,13 @@ private static Object extractValue(String[] pathElements, int index, Object curr
      * Only keep properties in {@code map} that match the {@code includes} but
      * not the {@code excludes}. An empty list of includes is interpreted as a
      * wildcard while an empty list of excludes does not match anything.
-     *
+     * 
                      
      * If a property matches both an include and an exclude, then the exclude
      * wins.
-     *
+     * 
                      
      * If an object matches, then any of its sub properties are automatically
      * considered as matching as well, both for includes and excludes.
-     *
+     * 
                      
      * Dots in field names are treated as sub objects. So for instance if a
      * document contains {@code a.b} as a property and {@code a} is an include,
      * then {@code a.b} will be kept in the filtered map.
@@ -555,7 +554,7 @@ public static Map nodeMapValue(Object node, String desc) {
 
     /**
      * Returns an array of string value from a node value.
-     *
+     * 
                      
      * If the node represents an array the corresponding array of strings is returned.
      * Otherwise the node is treated as a comma-separated string.
      */
diff --git a/server/src/main/java/org/opensearch/discovery/FileBasedSeedHostsProvider.java b/server/src/main/java/org/opensearch/discovery/FileBasedSeedHostsProvider.java
index 3159733336057..b663227978e8f 100644
--- a/server/src/main/java/org/opensearch/discovery/FileBasedSeedHostsProvider.java
+++ b/server/src/main/java/org/opensearch/discovery/FileBasedSeedHostsProvider.java
@@ -48,12 +48,12 @@
 /**
  * An implementation of {@link SeedHostsProvider} that reads hosts/ports
  * from {@link #UNICAST_HOSTS_FILE}.
- *
+ * 
                      
  * Each host/port that is part of the discovery process must be listed on
  * a separate line. If the port is left off an entry, we default to the
  * first port in the {@code transport.port} range.
  * An example unicast hosts file could read:
- *
+ * 
                      
  * 67.81.244.10
  * 67.81.244.11:9305
  * 67.81.244.15:9400
diff --git a/server/src/main/java/org/opensearch/discovery/SettingsBasedSeedHostsProvider.java b/server/src/main/java/org/opensearch/discovery/SettingsBasedSeedHostsProvider.java
index 9785d5b21078e..10185322c2ca6 100644
--- a/server/src/main/java/org/opensearch/discovery/SettingsBasedSeedHostsProvider.java
+++ b/server/src/main/java/org/opensearch/discovery/SettingsBasedSeedHostsProvider.java
@@ -49,7 +49,7 @@
  * An implementation of {@link SeedHostsProvider} that reads hosts/ports
  * from the "discovery.seed_hosts" node setting. If the port is
  * left off an entry, we default to the first port in the {@code transport.port} range.
- *
+ * 
                      
  * An example setting might look as follows:
  * [67.81.244.10, 67.81.244.11:9305, 67.81.244.15:9400]
  *
diff --git a/server/src/main/java/org/opensearch/env/Environment.java b/server/src/main/java/org/opensearch/env/Environment.java
index 3b87c756ffdae..5a40e45fb22a6 100644
--- a/server/src/main/java/org/opensearch/env/Environment.java
+++ b/server/src/main/java/org/opensearch/env/Environment.java
@@ -249,7 +249,7 @@ public Path[] repoFiles() {
 
     /**
      * Resolves the specified location against the list of configured repository roots
-     *
+     * 
                      
      * If the specified location doesn't match any of the roots, returns null.
      */
     public Path resolveRepoFile(String location) {
@@ -259,7 +259,7 @@ public Path resolveRepoFile(String location) {
     /**
      * Checks if the specified URL is pointing to the local file system and if it does, resolves the specified url
      * against the list of configured repository roots
-     *
+     * 
                      
      * If the specified url doesn't match any of the roots, returns null.
      */
     public URL resolveRepoURL(URL url) {
diff --git a/server/src/main/java/org/opensearch/env/NodeEnvironment.java b/server/src/main/java/org/opensearch/env/NodeEnvironment.java
index 3c5ab5ba98875..40c5722308b79 100644
--- a/server/src/main/java/org/opensearch/env/NodeEnvironment.java
+++ b/server/src/main/java/org/opensearch/env/NodeEnvironment.java
@@ -705,7 +705,7 @@ public List lockAllForIndex(
      * write operation on a shards data directory like deleting files, creating a new index writer
      * or recover from a different shard instance into it. If the shard lock can not be acquired
      * a {@link ShardLockObtainFailedException} is thrown.
-     *
+     * 
                      
      * Note: this method will return immediately if the lock can't be acquired.
      *
      * @param id the shard ID to lock
@@ -778,7 +778,7 @@ public interface ShardLocker {
 
     /**
      * Returns all currently lock shards.
-     *
+     * 
                      
      * Note: the shard ids return do not contain a valid Index UUID
      */
     public Set lockedShards() {
diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionReader.java b/server/src/main/java/org/opensearch/extensions/ExtensionReader.java
index 4556d6537ffdc..fc2ba817ace5f 100644
--- a/server/src/main/java/org/opensearch/extensions/ExtensionReader.java
+++ b/server/src/main/java/org/opensearch/extensions/ExtensionReader.java
@@ -17,7 +17,7 @@
  * a category class used to identify the reader defined within the JVM that the extension is running on.
  * Additionally, this method takes in the extension's corresponding DiscoveryNode and a byte array (context) that the
  * extension's reader will be applied to.
- *
+ * 
                      
  * By convention the extensions' reader is a constructor that takes StreamInput as an argument for most classes and a static method for things like enums.
  * Classes will implement this via a constructor (or a static method in the case of enumerations), it's something that should
  * look like:
diff --git a/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java b/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java
index 59ef894958cbe..853fe03904c53 100644
--- a/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java
@@ -50,7 +50,7 @@
 /**
  * An abstract class that implements basic functionality for allocating
  * shards to nodes based on shard copies that already exist in the cluster.
- *
+ * 
                      
  * Individual implementations of this class are responsible for providing
  * the logic to determine to which nodes (if any) those shards are allocated.
  *
diff --git a/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java b/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java
index e7c1ba01e7920..48479691689e5 100644
--- a/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java
+++ b/server/src/main/java/org/opensearch/gateway/DanglingIndicesState.java
@@ -244,7 +244,7 @@ List filterDanglingIndices(Metadata metadata, Map
      * Dangling importing indices with aliases is dangerous, it could for instance result in inability to write to an existing alias if it
      * previously had only one index with any is_write_index indication.
      */
diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
index 6b26af148b2ea..d3e7a0c482ee2 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -91,7 +91,7 @@
 
 /**
  * Loads (and maybe upgrades) cluster metadata at startup, and persistently stores cluster metadata for future restarts.
- *
+ * 
                      
  * When started, ensures that this version is compatible with the state stored on disk, and performs a state upgrade if necessary. Note that the state being
  * loaded when constructing the instance of this class is not necessarily the state that will be used as {@link ClusterState#metadata()} because it might be
  * stale or incomplete. Cluster-manager-eligible nodes must perform an election to find a complete and non-stale state, and cluster-manager-ineligible nodes
diff --git a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java
index 8db4736bcdc40..75beb6e29599c 100644
--- a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java
@@ -111,16 +111,16 @@
  * Stores cluster metadata in a bare Lucene index (per data path) split across a number of documents. This is used by cluster-manager-eligible nodes
  * to record the last-accepted cluster state during publication. The metadata is written incrementally where possible, leaving alone any
  * documents that have not changed. The index has the following fields:
- *
+ * 
                      
  * +------------------------------+-----------------------------+----------------------------------------------+
  * | "type" (string field)        | "index_uuid" (string field) | "data" (stored binary field in SMILE format) |
  * +------------------------------+-----------------------------+----------------------------------------------+
  * | GLOBAL_TYPE_NAME == "global" | (omitted)                   | Global metadata                              |
  * | INDEX_TYPE_NAME  == "index"  | Index UUID                  | Index metadata                               |
  * +------------------------------+-----------------------------+----------------------------------------------+
- *
+ * 
                      
  * Additionally each commit has the following user data:
- *
+ * 
                      
  * +---------------------------+-------------------------+-------------------------------------------------------------------------------+
  * |        Key symbol         |       Key literal       |                                     Value                                     |
  * +---------------------------+-------------------------+-------------------------------------------------------------------------------+
@@ -129,7 +129,7 @@
  * | NODE_ID_KEY               | "node_id"               | The (persistent) ID of the node that wrote this metadata                      |
  * | NODE_VERSION_KEY          | "node_version"          | The (ID of the) version of the node that wrote this metadata                  |
  * +---------------------------+-------------------------+-------------------------------------------------------------------------------+
- *
+ * 
                      
  * (the last-accepted term is recorded in Metadata → CoordinationMetadata so does not need repeating here)
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java b/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java
index 4dc9396751fc9..2807be00feeaa 100644
--- a/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java
@@ -69,7 +69,7 @@
  * that holds a copy of the shard.  The shard metadata from each node is compared against the
  * set of valid allocation IDs and for all valid shard copies (if any), the primary shard allocator
  * executes the allocation deciders to chose a copy to assign the primary shard to.
- *
+ * 
                      
  * Note that the PrimaryShardAllocator does *not* allocate primaries on index creation
  * (see {@link org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator}),
  * nor does it allocate primaries when a primary shard failed and there is a valid replica
@@ -386,11 +386,11 @@ protected static NodeShardsResult buildNodeShardsResult(
             }
         }
 
-        /**
-         * Orders the active shards copies based on below comparators
-         * 1. No store exception i.e. shard copy is readable
-         * 2. Prefer previous primary shard
-         * 3. Prefer shard copy with the highest replication checkpoint. It is NO-OP for doc rep enabled indices.
+        /*
+          Orders the active shards copies based on below comparators
+          1. No store exception i.e. shard copy is readable
+          2. Prefer previous primary shard
+          3. Prefer shard copy with the highest replication checkpoint. It is NO-OP for doc rep enabled indices.
          */
         final Comparator comparator; // allocation preference
         if (matchAnyShard) {
diff --git a/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java b/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java
index a6c9319b6dc54..f530052c5bcd1 100644
--- a/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java
@@ -295,7 +295,7 @@ public AllocateUnassignedDecision makeAllocationDecision(
 
     /**
      * Determines if the shard can be allocated on at least one node based on the allocation deciders.
-     *
+     * 
                      
      * Returns the best allocation decision for allocating the shard on any node (i.e. YES if at least one
      * node decided YES, THROTTLE if at least one node decided THROTTLE, and NO if none of the nodes decided
      * YES or THROTTLE).  If in explain mode, also returns the node-level explanations as the second element
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 164611816e1dc..4a8a0618ffa60 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -735,10 +735,11 @@ private boolean isInvalidClusterUUID(ClusterMetadataManifest manifest) {
      */
     private List getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException {
         try {
-            /**
-             * {@link BlobContainer#listBlobsByPrefixInSortedOrder} will list the latest manifest file first
-             * as the manifest file name generated via {@link RemoteClusterStateService#getManifestFileName} ensures
-             * when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top.
+
+            /*
+              {@link BlobContainer#listBlobsByPrefixInSortedOrder} will list the latest manifest file first
+              as the manifest file name generated via {@link RemoteClusterStateService#getManifestFileName} ensures
+              when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top.
              */
             return manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder(
                 MANIFEST_FILE_PREFIX + DELIMITER,
diff --git a/server/src/main/java/org/opensearch/http/CorsHandler.java b/server/src/main/java/org/opensearch/http/CorsHandler.java
index 862c50ae6ac1f..464ced184d10e 100644
--- a/server/src/main/java/org/opensearch/http/CorsHandler.java
+++ b/server/src/main/java/org/opensearch/http/CorsHandler.java
@@ -81,7 +81,7 @@
  * This file is forked from the https://netty.io project. In particular it combines the following three
  * files: io.netty.handler.codec.http.cors.CorsHandler, io.netty.handler.codec.http.cors.CorsConfig, and
  * io.netty.handler.codec.http.cors.CorsConfigBuilder.
- *
+ * 
                      
  * It modifies the original netty code to operate on OpenSearch http request/response abstractions.
  * Additionally, it removes CORS features that are not used by OpenSearch.
  *
diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopIdentityPlugin.java b/server/src/main/java/org/opensearch/identity/noop/NoopIdentityPlugin.java
index c2367383df595..090b1f1d025e0 100644
--- a/server/src/main/java/org/opensearch/identity/noop/NoopIdentityPlugin.java
+++ b/server/src/main/java/org/opensearch/identity/noop/NoopIdentityPlugin.java
@@ -14,7 +14,7 @@
 
 /**
  * Implementation of identity plugin that does not enforce authentication or authorization
- *
+ * 
                      
  * This class and related classes in this package will not return nulls or fail access checks
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopSubject.java b/server/src/main/java/org/opensearch/identity/noop/NoopSubject.java
index 424a10204aa19..964a218db3cf5 100644
--- a/server/src/main/java/org/opensearch/identity/noop/NoopSubject.java
+++ b/server/src/main/java/org/opensearch/identity/noop/NoopSubject.java
@@ -17,7 +17,7 @@
 
 /**
  * Implementation of subject that is always authenticated
- *
+ * 
                      
  * This class and related classes in this package will not return nulls or fail permissions checks
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java b/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java
index 193966001f44c..4bd3ebdded588 100644
--- a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java
+++ b/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java
@@ -26,7 +26,7 @@ public class RestTokenExtractor {
 
     /**
      * Given a rest request it will extract authentication token
-     *
+     * 
                      
      * If no token was found, returns null.
      */
     public static AuthToken extractToken(final RestRequest request) {
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
index a53dbe246fa44..84e8e2f41aaf1 100644
--- a/server/src/main/java/org/opensearch/index/IndexService.java
+++ b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -696,7 +696,7 @@ public IndexSettings getIndexSettings() {
 
     /**
      * Creates a new QueryShardContext.
-     *
+     * 
                      
      * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
      * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
      */
@@ -706,7 +706,7 @@ public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searche
 
     /**
      * Creates a new QueryShardContext.
-     *
+     * 
                      
      * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
      * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
      */
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index ce6c1a5ad6284..e90e9259f6a5c 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -550,7 +550,7 @@ public static IndexMergePolicy fromString(String text) {
 
     /**
      * This setting controls if unreferenced files will be cleaned up in case segment merge fails due to disk full.
-     *
+     * 
                      
      * Defaults to true which means unreferenced files will be cleaned up in case segment merge fails.
      */
     public static final Setting INDEX_UNREFERENCED_FILE_CLEANUP = Setting.boolSetting(
@@ -566,7 +566,7 @@ public static IndexMergePolicy fromString(String text) {
      * documents) on the grounds that a file-based peer recovery may copy all of the documents in the shard over to the new peer, but is
      * significantly faster than replaying the missing operations on the peer, so once a peer falls far enough behind the primary it makes
      * more sense to copy all the data over again instead of replaying history.
-     *
+     * 
                      
      * Defaults to retaining history for up to 10% of the documents in the shard. This can only be changed in tests, since this setting is
      * intentionally unregistered.
      */
diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
index 763ed34a1e6a6..9edb268a5126c 100644
--- a/server/src/main/java/org/opensearch/index/IndexSortConfig.java
+++ b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
@@ -53,7 +53,7 @@
 
 /**
  * Holds all the information that is used to build the sort order of an index.
- *
+ * 
                      
  * The index sort settings are final and can be defined only at index creation.
  * These settings are divided in four lists that are merged during the initialization of this class:
  * 
                        
diff --git a/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java b/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java
index 9e170b448d641..b2b7781a20d26 100644
--- a/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java
+++ b/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java
@@ -48,14 +48,14 @@
  *
  * 
                          
  * 
                        •  index.merge.scheduler.max_thread_count:
- *
+ * 
                          
  *     The maximum number of threads that may be merging at once. Defaults to
  *     Math.max(1, Math.min(4, {@link OpenSearchExecutors#allocatedProcessors(Settings)} / 2))
  *     which works well for a good solid-state-disk (SSD).  If your index is on
  *     spinning platter drives instead, decrease this to 1.
  *
  * 
                        • index.merge.scheduler.auto_throttle:
- *
+ * 
                          
  *     If this is true (the default), then the merge scheduler will rate-limit IO
  *     (writes) for merges to an adaptive value depending on how many merges are
  *     requested over time.  An application with a low indexing rate that
diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java
index 07768c6769b71..a6135186fb5ff 100644
--- a/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java
+++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressure.java
@@ -34,7 +34,7 @@
  * Interfaces returns Releasable which when triggered will release the acquired accounting tokens values and also
  * perform necessary actions such as throughput evaluation once the request completes.
  * Consumers of these interfaces are expected to trigger close on releasable, reliably for consistency.
- *
+ * 
                          
  * Overall ShardIndexingPressure provides:
  * 1. Memory Accounting at shard level. This can be enabled/disabled based on dynamic setting.
  * 2. Memory Accounting at Node level. Tracking is done using the IndexingPressure artefacts to support feature seamless toggling.
diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java
index dea3cc8970cb5..e5c1af2e9c9f0 100644
--- a/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java
+++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressureMemoryManager.java
@@ -30,16 +30,16 @@
 /**
  * The Shard Indexing Pressure Memory Manager is the construct responsible for increasing and decreasing the allocated shard limit
  * based on incoming requests. A shard limits defines the maximum memory that a shard can occupy in the heap for request objects.
- *
+ * 
                          
  * Based on the overall memory utilization on the node, and current traffic needs shard limits will be modified:
- *
+ * 
                          
  * 1. If the limits assigned to a shard is breached (Primary Parameter) while the node level overall occupancy across all shards
  * is not greater than primary_parameter.node.soft_limit, MemoryManager will increase the shard limits without any deeper evaluation.
  * 2. If the limits assigned to the shard is breached(Primary Parameter) and the node level overall occupancy across all shards
  * is greater than primary_parameter.node.soft_limit, then MemoryManager will evaluate deeper parameters for shards to identify any
  * issues, such as throughput degradation (Secondary Parameter - 1) and time since last request was successful (Secondary Parameter - 2).
  * This helps identify detect any duress state with the shard, requesting more memory.
- *
+ * 
                          
  * Secondary Parameters covered above:
  * 1. ThroughputDegradationLimitsBreached - When the moving window throughput average has increased by a factor compared to
  * the historical throughput average. If the factor by which it has increased is greater than the degradation limit threshold, this
@@ -47,7 +47,7 @@
  * 2. LastSuccessfulRequestDurationLimitsBreached - When the time since the last successful request completed is greater than the max
  * timeout threshold value, while there a number of outstanding requests greater than the max outstanding requests then this parameter
  * is considered to be breached.
- *
+ * 
                          
  * MemoryManager attempts to increase of decrease the shard limits in case the shard utilization goes below operating_factor.lower or
  * goes above operating_factor.upper of current shard limits. MemoryManager attempts to update the new shard limit such that the new value
  * remains withing the operating_factor.optimal range of current shard utilization.
diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java
index b41dd1359394b..9b24d119f24fd 100644
--- a/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java
+++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressureStore.java
@@ -22,24 +22,24 @@
  * Shard indexing pressure store acts as a central repository for all the shard-level tracker objects currently being
  * used at the Node level, for tracking indexing pressure requests.
  * Store manages the tracker lifecycle, from creation, access, until it is evicted to be collected.
- *
+ * 
                          
  * Trackers are maintained at two levels for access simplicity and better memory management:
- *
+ * 
                          
  * 1. shardIndexingPressureHotStore : As the name suggests, it is hot store for tracker objects which are currently live i.e. being used
  * to track an ongoing request.
- *
+ * 
                          
  * 2. shardIndexingPressureColdStore : This acts as the store for all the shard tracking objects which are currently being used
  * by the framework. In addition to hot trackers, the recently used trackers which are although not currently live, but again can be used
  * in near future, are also part of this store. To limit any memory implications, this store has an upper limit on the maximum number of
  * trackers its can hold at any given time, which is a configurable dynamic setting.
- *
+ * 
                          
  * Tracking objects when created are part of both the hot store as well as cold store. However, once the object
  * is no more live it is removed from the hot store. Objects in the cold store are evicted once the cold store
  * reaches its maximum limit. Think of it like a periodic purge when upper limit is hit.
  * During get if tracking object is not present in the hot store, a lookup is made into the cache store. If found,
  * object is brought into the hot store again, until it remains active. If not present in the either store, a fresh
  * object is instantiated and registered in both the stores for concurrent accesses.
- *
+ * 
                          
  * Note: The implementation of shardIndexingPressureColdStore methods is such that get,
  * update and evict operations can be abstracted out to support any other strategy such as LRU, if
  * discovered a need later.
diff --git a/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java b/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java
index 7d67b47141ef5..e0edb8260fd0f 100644
--- a/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java
+++ b/server/src/main/java/org/opensearch/index/ShardIndexingPressureTracker.java
@@ -14,19 +14,19 @@
  * This class is responsible for all the tracking that needs to be performed at every Shard Level for Indexing Operations on the node.
  * Info is maintained at the granularity of three kinds of write operation (tasks) on the node i.e. coordinating, primary and replica.
  * This is useful in evaluating the shard indexing back-pressure on the node, to throttle requests and also to publish runtime stats.
- *
+ * 
                          
  * There can be four kinds of operation tracking on a node which needs to performed for a shard:
  * 1. Coordinating Operation : To track all the individual shard bulk request on the coordinator node.
  * 2. Primary Operation : To track all the individual shard bulk request on the primary node.
  * 3. Replica Operation : To track all the individual shard bulk request on the replica node.
  * 4. Common Operation : To track values applicable across the specific shard role.
- *
+ * 
                          
  * ShardIndexingPressureTracker therefore provides the construct to track all the write requests targeted for a ShardId on the node,
  * across all possible transport-write-actions i.e. Coordinator, Primary and Replica.
  * Tracker is uniquely identified against a Shard-Id on the node. Currently the knowledge of shard roles (such as primary vs replica)
  * is not explicit to the tracker, and it is able to track different values simultaneously based on the interaction hooks of the
  * operation type i.e. write-action layers.
- *
+ * 
                          
  * There is room for introducing more unique identity to the trackers based on Shard-Role or Shard-Allocation-Id, but that will also
  * increase the complexity of handling shard-lister events and handling other race scenarios such as request-draining etc.
  * To prefer simplicity we have modelled by keeping explicit fields for different operation tracking, while tracker by itself is
diff --git a/server/src/main/java/org/opensearch/index/VersionType.java b/server/src/main/java/org/opensearch/index/VersionType.java
index 3ce7f600a6a5b..8aa1fbd1b81ac 100644
--- a/server/src/main/java/org/opensearch/index/VersionType.java
+++ b/server/src/main/java/org/opensearch/index/VersionType.java
@@ -244,7 +244,7 @@ public byte getValue() {
 
     /**
      * Returns a human readable explanation for a version conflict on write.
-     *
+     * 
                          
      * Note that this method is only called if {@link #isVersionConflictForWrites(long, long, boolean)} returns true;
      *
      * @param currentVersion  the current version for the document
@@ -265,7 +265,7 @@ public byte getValue() {
 
     /**
      * Returns a human readable explanation for a version conflict on read.
-     *
+     * 
                          
      * Note that this method is only called if {@link #isVersionConflictForReads(long, long)} returns true;
      *
      * @param currentVersion  the current version for the document
diff --git a/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java
index cfdf416b2d533..3b1f6af8a0030 100644
--- a/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java
+++ b/server/src/main/java/org/opensearch/index/analysis/AnalysisRegistry.java
@@ -244,7 +244,7 @@ public IndexAnalyzers build(IndexSettings indexSettings) throws IOException {
 
     /**
      * Creates a custom analyzer from a collection of {@link NameOrDefinition} specifications for each component
-     *
+     * 
                          
      * Callers are responsible for closing the returned Analyzer
      */
     public NamedAnalyzer buildCustomAnalyzer(
diff --git a/server/src/main/java/org/opensearch/index/analysis/NormalizingCharFilterFactory.java b/server/src/main/java/org/opensearch/index/analysis/NormalizingCharFilterFactory.java
index ab8d23339029c..30fe31105e1d9 100644
--- a/server/src/main/java/org/opensearch/index/analysis/NormalizingCharFilterFactory.java
+++ b/server/src/main/java/org/opensearch/index/analysis/NormalizingCharFilterFactory.java
@@ -36,7 +36,7 @@
 
 /**
  * A CharFilterFactory that also supports normalization
- *
+ * 
                          
  * The default implementation of {@link #normalize(Reader)} delegates to
  * {@link #create(Reader)}
  *
diff --git a/server/src/main/java/org/opensearch/index/analysis/NormalizingTokenFilterFactory.java b/server/src/main/java/org/opensearch/index/analysis/NormalizingTokenFilterFactory.java
index be761aee0d36c..2ed621cdd22b1 100644
--- a/server/src/main/java/org/opensearch/index/analysis/NormalizingTokenFilterFactory.java
+++ b/server/src/main/java/org/opensearch/index/analysis/NormalizingTokenFilterFactory.java
@@ -36,7 +36,7 @@
 
 /**
  * A TokenFilterFactory that may be used for normalization
- *
+ * 
                          
  * The default implementation delegates {@link #normalize(TokenStream)} to
  * {@link #create(TokenStream)}}.
  *
diff --git a/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java
index a65e1898bea0d..8719d127781e0 100644
--- a/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java
+++ b/server/src/main/java/org/opensearch/index/analysis/PreBuiltAnalyzerProviderFactory.java
@@ -102,7 +102,7 @@ public void close() throws IOException {
 
     /**
      *  A special cache that closes the gap between PreBuiltAnalyzers and PreBuiltAnalyzerProviderFactory.
-     *
+     * 
                          
      *  This can be removed when all analyzers have been moved away from PreBuiltAnalyzers to
      *  PreBuiltAnalyzerProviderFactory either in server or analysis-common.
      *
diff --git a/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java b/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java
index e66ae20508dfe..16a1f9a067998 100644
--- a/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java
+++ b/server/src/main/java/org/opensearch/index/analysis/ShingleTokenFilterFactory.java
@@ -155,11 +155,11 @@ public TokenStream create(TokenStream tokenStream) {
             filter.setTokenSeparator(tokenSeparator);
             filter.setFillerToken(fillerToken);
             if (outputUnigrams || (minShingleSize != maxShingleSize)) {
-                /**
-                 * We disable the graph analysis on this token stream
-                 * because it produces shingles of different size.
-                 * Graph analysis on such token stream is useless and dangerous as it may create too many paths
-                 * since shingles of different size are not aligned in terms of positions.
+                /*
+                  We disable the graph analysis on this token stream
+                  because it produces shingles of different size.
+                  Graph analysis on such token stream is useless and dangerous as it may create too many paths
+                  since shingles of different size are not aligned in terms of positions.
                  */
                 filter.addAttribute(DisableGraphAttribute.class);
             }
diff --git a/server/src/main/java/org/opensearch/index/analysis/TokenFilterFactory.java b/server/src/main/java/org/opensearch/index/analysis/TokenFilterFactory.java
index 1b9d781b177ce..6708db8571b2a 100644
--- a/server/src/main/java/org/opensearch/index/analysis/TokenFilterFactory.java
+++ b/server/src/main/java/org/opensearch/index/analysis/TokenFilterFactory.java
@@ -51,7 +51,7 @@ public interface TokenFilterFactory {
 
     /**
      * Normalize a tokenStream for use in multi-term queries
-     *
+     * 
                          
      * The default implementation is a no-op
      */
     default TokenStream normalize(TokenStream tokenStream) {
@@ -86,7 +86,7 @@ default TokenFilterFactory getChainAwareTokenFilterFactory(
 
     /**
      * Return a version of this TokenFilterFactory appropriate for synonym parsing
-     *
+     * 
                          
      * Filters that should not be applied to synonyms (for example, those that produce
      * multiple tokens) should throw an exception
      *
diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
index 3eeceff2253c1..8e1627af274c5 100644
--- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
@@ -402,7 +402,7 @@ public CompletionStats completionStats(String... fieldNamePatterns) {
      * The main purpose for this is that if we have external refreshes happening we don't issue extra
      * refreshes to clear version map memory etc. this can cause excessive segment creation if heavy indexing
      * is happening and the refresh interval is low (ie. 1 sec)
-     *
+     * 
                          
      * This also prevents segment starvation where an internal reader holds on to old segments literally forever
      * since no indexing is happening and refreshes are only happening to the external reader manager, while with
      * this specialized implementation an external refresh will immediately be reflected on the internal reader
diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
index 020e92aba4ce5..f9f0458f45ee8 100644
--- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
@@ -180,7 +180,7 @@ public synchronized void updateSegments(final SegmentInfos infos) throws IOExcep
 
     /**
      * Persist the latest live SegmentInfos.
-     *
+     * 
                          
      * This method creates a commit point from the latest SegmentInfos.
      *
      * @throws IOException - When there is an IO error committing the SegmentInfos.
diff --git a/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java b/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java
index 2e8bd6409c2f6..19454967f9ee3 100644
--- a/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java
+++ b/server/src/main/java/org/opensearch/index/engine/ReplicaFileTracker.java
@@ -22,7 +22,7 @@
 /**
  * This class is heavily influenced by Lucene's ReplicaFileDeleter class used to keep track of
  * segment files that should be preserved on replicas between replication events.
- *
+ * 
                          
  * https://github.com/apache/lucene/blob/main/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/ReplicaFileDeleter.java
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/index/fielddata/MultiGeoPointValues.java b/server/src/main/java/org/opensearch/index/fielddata/MultiGeoPointValues.java
index 2df4baeb8631b..3090b8e7f5b15 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/MultiGeoPointValues.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/MultiGeoPointValues.java
@@ -73,7 +73,7 @@ protected MultiGeoPointValues() {}
     /**
      * Return the next value associated with the current document. This must not be
      * called more than {@link #docValueCount()} times.
-     *
+     * 
                          
      * Note: the returned {@link GeoPoint} might be shared across invocations.
      *
      * @return the next value for the current docID set to {@link #advanceExact(int)}.
diff --git a/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java
index a3c3774b250dc..5860cfe8a47c3 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java
@@ -55,7 +55,7 @@
 /**
  * Script level doc values, the assumption is that any implementation will
  * implement a {@link Longs#getValue getValue} method.
- *
+ * 
                          
  * Implementations should not internally re-use objects for the values that they
  * return as a single {@link ScriptDocValues} instance can be reused to return
  * values form multiple documents.
@@ -589,11 +589,11 @@ public BytesRef get(int index) {
                         + "Use doc[].size()==0 to check if a document is missing a field!"
                 );
             }
-            /**
-             * We need to make a copy here because {@link BinaryScriptDocValues} might reuse the
-             * returned value and the same instance might be used to
-             * return values from multiple documents.
-             **/
+            /*
+              We need to make a copy here because {@link BinaryScriptDocValues} might reuse the
+              returned value and the same instance might be used to
+              return values from multiple documents.
+             */
             return values[index].toBytesRef();
         }
 
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java
index 4f27c9b10f0ee..fe033fa7a3f70 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java
@@ -101,9 +101,9 @@ public SortedSetOrdinalsIndexFieldData(
     @Override
     public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
         XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
-        /**
-         * Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
-         * returns a custom sort field otherwise.
+        /*
+          Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
+          returns a custom sort field otherwise.
          */
         if (nested != null
             || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN)
diff --git a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java
index a038cf178bb03..3b6782b34feea 100644
--- a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java
@@ -121,7 +121,7 @@ public abstract static class Parser {
 
         /**
          * Given a parsed value and a format string, formats the value into a plain Java object.
-         *
+         * 
                          
          * Supported formats include 'geojson' and 'wkt'. The different formats are defined
          * as subclasses of {@link org.opensearch.common.geo.GeometryFormat}.
          */
@@ -129,7 +129,7 @@ public abstract static class Parser {
 
         /**
          * Parses the given value, then formats it according to the 'format' string.
-         *
+         * 
                          
          * By default, this method simply parses the value using {@link Parser#parse}, then formats
          * it with {@link Parser#format}. However some {@link Parser} implementations override this
          * as they can avoid parsing the value if it is already in the right format.
diff --git a/server/src/main/java/org/opensearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/opensearch/index/mapper/ArraySourceValueFetcher.java
index f5dc34ab8ac5d..b3112df86bab6 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ArraySourceValueFetcher.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ArraySourceValueFetcher.java
@@ -43,7 +43,7 @@
 /**
  * An implementation of {@link ValueFetcher} that knows how to extract values
  * from the document source.
- *
+ * 
                          
  * This class differs from {@link SourceValueFetcher} in that it directly handles
  * array values in parsing. Field types should use this class if their corresponding
  * mapper returns true for {@link FieldMapper#parsesArrayValue()}.
diff --git a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java
index 90ef3cc1689a4..e1413fd9b4bbe 100644
--- a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java
@@ -447,13 +447,13 @@ int getMaxInputLength() {
 
     /**
      * Parses and indexes inputs
-     *
+     * 
                          
      * Parsing:
      *  Acceptable format:
      *   "STRING" - interpreted as field value (input)
      *   "ARRAY" - each element can be one of "OBJECT" (see below)
      *   "OBJECT" - { "input": STRING|ARRAY, "weight": STRING|INT, "contexts": ARRAY|OBJECT }
-     *
+     * 
                          
      * Indexing:
      *  if context mappings are defined, delegates to {@link ContextMappings#addField(ParseContext.Document, String, String, int, Map)}
      *  else adds inputs as a {@link org.apache.lucene.search.suggest.document.SuggestField}
diff --git a/server/src/main/java/org/opensearch/index/mapper/CustomDocValuesField.java b/server/src/main/java/org/opensearch/index/mapper/CustomDocValuesField.java
index 2776e7515bbf6..fbb67731f581b 100644
--- a/server/src/main/java/org/opensearch/index/mapper/CustomDocValuesField.java
+++ b/server/src/main/java/org/opensearch/index/mapper/CustomDocValuesField.java
@@ -42,8 +42,7 @@
 import java.io.Reader;
 
 /**
- * Base class for constructing a custom docvalues type
- *
+ * Base class for constructing a custom docvalues type.
  * used for binary, geo, and range fields
  *
  * @opensearch.api
diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
index dbafc2a923cf2..db58c5e02e176 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
@@ -197,7 +197,7 @@ private static String getRemainingFields(Map map) {
 
     /**
      * Given an optional type name and mapping definition, returns the type and a normalized form of the mappings.
-     *
+     * 
                          
      * The provided mapping definition may or may not contain the type name as the root key in the map. This method
      * attempts to unwrap the mappings, so that they no longer contain a type name at the root. If no type name can
      * be found, through either the 'type' parameter or by examining the provided mappings, then an exception will be
@@ -205,7 +205,6 @@ private static String getRemainingFields(Map map) {
      *
      * @param type An optional type name.
      * @param root The mapping definition.
-     *
      * @return A tuple of the form (type, normalized mappings).
      */
     @SuppressWarnings({ "unchecked" })
diff --git a/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldMapper.java
index 94bc4806ba0e0..2e59d86f9119c 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldMapper.java
@@ -43,7 +43,7 @@
  *   to {@link DynamicKeyFieldMapper#keyedFieldType(String)}, with 'some_key' passed as the
  *   argument. The field mapper is allowed to create a new field type dynamically in order
  *   to handle the search.
- *
+ * 
                          
  * To prevent conflicts between these dynamic sub-keys and multi-fields, any field mappers
  * implementing this interface should explicitly disallow multi-fields. The constructor makes
  * sure to passes an empty multi-fields list to help prevent conflicting sub-keys from being
diff --git a/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldTypeLookup.java b/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldTypeLookup.java
index 13150ddc50a51..a415078108eb6 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldTypeLookup.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DynamicKeyFieldTypeLookup.java
@@ -37,11 +37,11 @@
 
 /**
  * A container that supports looking up field types for 'dynamic key' fields ({@link DynamicKeyFieldMapper}).
- *
+ * 
                          
  * Compared to standard fields, 'dynamic key' fields require special handling. Given a field name of the form
  * 'path_to_field.path_to_key', the container will dynamically return a new {@link MappedFieldType} that is
  * suitable for performing searches on the sub-key.
- *
+ * 
                          
  * Note: we anticipate that 'flattened' fields will be the only implementation {@link DynamicKeyFieldMapper}.
  * Flattened object fields live in the 'mapper-flattened' module.
  *
diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldAliasMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldAliasMapper.java
index 86c92ec19a2f7..ff9cb61b85571 100644
--- a/server/src/main/java/org/opensearch/index/mapper/FieldAliasMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/FieldAliasMapper.java
@@ -43,7 +43,7 @@
 
 /**
  * A mapper for field aliases.
- *
+ * 
                          
  * A field alias has no concrete field mappings of its own, but instead points to another field by
  * its path. Once defined, an alias can be used in place of the concrete field name in search requests.
  *
diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java
index f8514c86fa418..f6178a8284945 100644
--- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java
@@ -305,7 +305,7 @@ public void parse(ParseContext context) throws IOException {
 
     /**
      * Parse the field value and populate the fields on {@link ParseContext#doc()}.
-     *
+     * 
                          
      * Implementations of this method should ensure that on failing to parse parser.currentToken() must be the
      * current failing token
      */
diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java
index 8e1b6f2a3c08b..549ab0403e5dc 100644
--- a/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java
+++ b/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java
@@ -57,7 +57,7 @@ class FieldTypeLookup implements Iterable {
      * A map from field name to all fields whose content has been copied into it
      * through copy_to. A field only be present in the map if some other field
      * has listed it as a target of copy_to.
-     *
+     * 
                          
      * For convenience, the set of copied fields includes the field itself.
      */
     private final Map> fieldToCopiedFields = new HashMap<>();
@@ -133,7 +133,7 @@ public Set simpleMatchToFullName(String pattern) {
 
     /**
      * Given a concrete field name, return its paths in the _source.
-     *
+     * 
                          
      * For most fields, the source path is the same as the field itself. However
      * there are cases where a field's values are found elsewhere in the _source:
      *   - For a multi-field, the source path is the parent field.
diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java
index db35c3edcc4a8..00b623dddac23 100644
--- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java
@@ -575,9 +575,9 @@ protected void parseCreateField(ParseContext context) throws IOException {
                 context,
                 fieldType().name()
             );
-            /**
-             * JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser
-             * It reads the JSON object and parsed to a list of string
+            /*
+              JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser
+              It reads the JSON object and parsed to a list of string
              */
             XContentParser parser = JsonToStringParser.parseObject();
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java
index e3dab3f892949..fcca7e9804bf3 100644
--- a/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java
@@ -63,7 +63,7 @@
 
 /**
  * Field Mapper for geo_point types.
- *
+ * 
                          
  * Uses lucene 6 LatLonPoint encoding
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java
index 58aa0bb2576e2..997835f712038 100644
--- a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java
@@ -125,7 +125,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
 
     /**
      * Create a helper class to fetch field values during the {@link FetchFieldsPhase}.
-     *
+     * 
                          
      * New field types must implement this method in order to support the search 'fields' option. Except
      * for metadata fields, field types should not throw {@link UnsupportedOperationException} since this
      * could cause a search retrieving multiple fields (like "fields": ["*"]) to fail.
@@ -487,7 +487,7 @@ public Map meta() {
 
     /**
      * Returns information on how any text in this field is indexed
-     *
+     * 
                          
      * Fields that do not support any text-based queries should return
      * {@link TextSearchInfo#NONE}.  Some fields (eg numeric) may support
      * only simple match queries, and can return
diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java
index 59c647d38f0de..9c0e3ef72549a 100644
--- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java
@@ -190,7 +190,7 @@ public Supplier queryShardContextSupplier() {
 
             /**
              * Gets an optional default date format for date fields that do not have an explicit format set
-             *
+             * 
                          
              * If {@code null}, then date fields will default to {@link DateFieldMapper#DEFAULT_DATE_TIME_FORMATTER}.
              */
             public DateFormatter getDateFormatter() {
diff --git a/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java b/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java
index 024f4b71584bf..1c608fc52c1f5 100644
--- a/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java
+++ b/server/src/main/java/org/opensearch/index/mapper/MappingLookup.java
@@ -157,7 +157,7 @@ public MappingLookup(
     /**
      * Returns the leaf mapper associated with this field name. Note that the returned mapper
      * could be either a concrete {@link FieldMapper}, or a {@link FieldAliasMapper}.
-     *
+     * 
                          
      * To access a field's type information, {@link MapperService#fieldType} should be used instead.
      */
     public Mapper getMapper(String field) {
diff --git a/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java
index 4f5aefdfeed55..5a45ab72994ee 100644
--- a/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java
@@ -70,7 +70,7 @@ public interface TypeParser extends Mapper.TypeParser {
 
     /**
      * Declares an updateable boolean parameter for a metadata field
-     *
+     * 
                          
      * We need to distinguish between explicit configuration and default value for metadata
      * fields, because mapping updates will carry over the previous metadata values if a
      * metadata field is not explicitly declared in the update.  A standard boolean
diff --git a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
index fd57975831e88..93b929a82f095 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
@@ -63,11 +63,11 @@
 
 /**
  * Defines how a particular field should be indexed and searched
- *
+ * 
                          
  * Configuration {@link Parameter}s for the mapper are defined on a {@link Builder} subclass,
  * and returned by its {@link Builder#getParameters()} method.  Merging, serialization
  * and parsing of the mapper are all mediated through this set of parameters.
- *
+ * 
                          
  * Subclasses should implement a {@link Builder} that is returned from the
  * {@link #getMergeBuilder()} method, initialised with the existing builder.
  *
@@ -86,7 +86,7 @@ protected ParametrizedFieldMapper(String simpleName, MappedFieldType mappedField
 
     /**
      * Returns a {@link Builder} to be used for merging and serialization
-     *
+     * 
                          
      * Implement as follows:
      * {@code return new MyBuilder(simpleName()).init(this); }
      */
@@ -256,7 +256,7 @@ public Parameter acceptsNull() {
 
         /**
          * Adds a deprecated parameter name.
-         *
+         * 
                          
          * If this parameter name is encountered during parsing, a deprecation warning will
          * be emitted.  The parameter will be serialized with its main name.
          */
diff --git a/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java
index 16f76f087e403..038066773a360 100644
--- a/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/SeqNoFieldMapper.java
@@ -55,11 +55,11 @@
 
 /**
  * Mapper for the {@code _seq_no} field.
- *
+ * 
                          
  * We expect to use the seq# for sorting, during collision checking and for
  * doing range searches. Therefore the {@code _seq_no} field is stored both
  * as a numeric doc value and as numeric indexed field.
- *
+ * 
                          
  * This mapper also manages the primary term field, which has no OpenSearch named
  * equivalent. The primary term is only used during collision after receiving
  * identical seq# values for two document copies. The primary term is stored as
diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/opensearch/index/mapper/SourceValueFetcher.java
index 69f53ba126790..a32d1c9f489ca 100644
--- a/server/src/main/java/org/opensearch/index/mapper/SourceValueFetcher.java
+++ b/server/src/main/java/org/opensearch/index/mapper/SourceValueFetcher.java
@@ -46,7 +46,7 @@
  * An implementation of {@link ValueFetcher} that knows how to extract values
  * from the document source. Most standard field mappers will use this class
  * to implement value fetching.
- *
+ * 
                          
  * Field types that handle arrays directly should instead use {@link ArraySourceValueFetcher}.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/opensearch/index/mapper/ValueFetcher.java
index d1cea3fe7f1b0..96237b16ea5a4 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ValueFetcher.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ValueFetcher.java
@@ -48,11 +48,11 @@
 public interface ValueFetcher {
     /**
      * Given access to a document's _source, return this field's values.
-     *
+     * 
                          
      * In addition to pulling out the values, they will be parsed into a standard form.
      * For example numeric field mappers make sure to parse the source value into a number
      * of the right type.
-     *
+     * 
                          
      * Note that for array values, the order in which values are returned is undefined and
      * should not be relied on.
      *
diff --git a/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java
index 6b8dd08ed0d91..66c6ee115c3f0 100644
--- a/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java
@@ -298,7 +298,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws I
 
     /**
      * For internal usage only!
-     *
+     * 
                          
      * Extracts the inner hits from the query tree.
      * While it extracts inner hits, child inner hits are inlined into the inner hit builder they belong to.
      */
diff --git a/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java
index a4b75beab26ea..1fade8601e2a6 100644
--- a/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -60,7 +60,7 @@
 /**
  * Creates a Lucene query that will filter for all documents that lie within the specified
  * bounding box.
- *
+ * 
                          
  * This query can only operate on fields of type geo_point that have latitude and longitude
  * enabled.
  *
diff --git a/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java
index e029884f32531..33b896a1d5163 100644
--- a/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/GeoShapeQueryBuilder.java
@@ -55,7 +55,7 @@
 /**
  * Derived {@link AbstractGeometryQueryBuilder} that builds a lat, lon GeoShape Query. It
  * can be applied to any {@link MappedFieldType} that implements {@link GeoShapeQueryable}.
- *
+ * 
                          
  * GeoJson and WKT shape definitions are supported
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java b/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java
index 559c084325abb..bb3cd34ae629d 100644
--- a/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java
+++ b/server/src/main/java/org/opensearch/index/query/GeoValidationMethod.java
@@ -42,7 +42,7 @@
 /**
  * This enum is used to determine how to deal with invalid geo coordinates in geo related
  * queries:
- *
+ * 
                          
  *  On STRICT validation invalid coordinates cause an exception to be thrown.
  *  On IGNORE_MALFORMED invalid coordinates are being accepted.
  *  On COERCE invalid coordinates are being corrected to the most likely valid coordinate.
diff --git a/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java
index b5cc3238295e0..5a57dfed14f69 100644
--- a/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java
+++ b/server/src/main/java/org/opensearch/index/query/IntervalsSourceProvider.java
@@ -691,7 +691,7 @@ public static class Regexp extends IntervalsSourceProvider {
 
         /**
          * Constructor
-         *
+         * 
                          
          * {@code flags} is Lucene's syntax flags
          * and {@code caseInsensitive} enables Lucene's only matching flag.
          */
diff --git a/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java
index 9a8d2ab104799..7ceb17203e837 100644
--- a/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java
@@ -176,7 +176,7 @@ public String minimumShouldMatch() {
     }
 
     @Deprecated
-    /** Sets the fuzziness used when evaluated to a fuzzy query type. Defaults to "AUTO". */
+    /* Sets the fuzziness used when evaluated to a fuzzy query type. Defaults to "AUTO". */
     public MatchBoolPrefixQueryBuilder fuzziness(Object fuzziness) {
         this.fuzziness = Fuzziness.build(fuzziness);
         return this;
diff --git a/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java
index 844d47070923b..5e9e6a3660e76 100644
--- a/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java
@@ -209,7 +209,7 @@ public String analyzer() {
     }
 
     @Deprecated
-    /** Sets the fuzziness used when evaluated to a fuzzy query type. Defaults to "AUTO". */
+    /* Sets the fuzziness used when evaluated to a fuzzy query type. Defaults to "AUTO". */
     public MatchQueryBuilder fuzziness(Object fuzziness) {
         this.fuzziness = Fuzziness.build(fuzziness);
         return this;
diff --git a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java
index 84b30209f31bd..e6472afef2215 100644
--- a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java
@@ -87,7 +87,7 @@
 
 /**
  * A more like this query that finds documents that are "like" the provided set of document(s).
- *
+ * 
                          
  * The documents are provided as a set of strings and/or a list of {@link Item}.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java
index 1aa013cb24d5e..6227e5d2fa806 100644
--- a/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java
@@ -398,8 +398,8 @@ public int slop() {
     }
 
     @Deprecated
-    /**
-     * Sets the fuzziness used when evaluated to a fuzzy query type. Defaults to "AUTO".
+    /*
+      Sets the fuzziness used when evaluated to a fuzzy query type. Defaults to "AUTO".
      */
     public MultiMatchQueryBuilder fuzziness(Object fuzziness) {
         if (fuzziness != null) {
diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
index 701484fbc8dc3..7b248c2a6f3c3 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
@@ -510,7 +510,7 @@ public final void freezeContext() {
     /**
      * This method fails if {@link #freezeContext()} is called before on this
      * context. This is used to seal.
-     *
+     * 
                          
      * This methods and all methods that call it should be final to ensure that
      * setting the request as not cacheable and the freezing behaviour of this
      * class cannot be bypassed. This is important so we can trust when this
diff --git a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java
index 469dc51da323a..3d8fbd5fc436d 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java
@@ -119,7 +119,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder
      * Can be changed back to HashMap once https://issues.apache.org/jira/browse/LUCENE-6305 is fixed.
      */
     private final Map fieldsAndWeights = new TreeMap<>();
diff --git a/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java
index 0d54373112904..fdbef2c732361 100644
--- a/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java
@@ -504,9 +504,9 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         if (from == null && to == null) {
-            /**
-             * Open bounds on both side, we can rewrite to an exists query
-             * if the {@link FieldNamesFieldMapper} is enabled.
+            /*
+              Open bounds on both side, we can rewrite to an exists query
+              if the {@link FieldNamesFieldMapper} is enabled.
              */
             final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType) context
                 .getMapperService()
diff --git a/server/src/main/java/org/opensearch/index/query/SearchIndexNameMatcher.java b/server/src/main/java/org/opensearch/index/query/SearchIndexNameMatcher.java
index 00758309fc0f0..598406b4e45f2 100644
--- a/server/src/main/java/org/opensearch/index/query/SearchIndexNameMatcher.java
+++ b/server/src/main/java/org/opensearch/index/query/SearchIndexNameMatcher.java
@@ -73,7 +73,7 @@ public SearchIndexNameMatcher(
 
     /**
      *  Given an index pattern, checks whether it matches against the current shard.
-     *
+     * 
                          
      *  If this shard represents a remote shard target, then in order to match the pattern contain
      *  the separator ':', and must match on both the cluster alias and index name.
      */
diff --git a/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java b/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java
index 3cd0554af49a4..57ae7dd0ea5e9 100644
--- a/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java
@@ -66,7 +66,7 @@
  * 
                        • '{@code -}' negates a single token: {@code -token0}
  * 
                        • '{@code "}' creates phrases of terms: {@code "term1 term2 ..."}
  * 
                        • '{@code *}' at the end of terms specifies prefix query: {@code term*}
- * 
                        • '{@code (}' and '{@code)}' specifies precedence: {@code token1 + (token2 | token3)}
+ * 
                        • '{@code (}' and '{@code )}' specifies precedence: {@code token1 + (token2 | token3)}
  * 
                        • '{@code ~}N' at the end of terms specifies fuzzy query: {@code term~1}
  * 
                        • '{@code ~}N' at the end of phrases specifies near/slop query: {@code "term1 term2"~5}
  * 
                        
diff --git a/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java b/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java
index f33800659245f..d7c0da4773fff 100644
--- a/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java
+++ b/server/src/main/java/org/opensearch/index/reindex/BulkByScrollTask.java
@@ -73,13 +73,13 @@
 
 /**
  * Task storing information about a currently running BulkByScroll request.
- *
+ * 
                        
  * When the request is not sliced, this task is the only task created, and starts an action to perform search requests.
- *
+ * 
                        
  * When the request is sliced, this task can either represent a coordinating task (using
  * {@link BulkByScrollTask#setWorkerCount(int)}) or a worker task that performs search queries (using
  * {@link BulkByScrollTask#setWorker(float, Integer)}).
- *
+ * 
                        
  * We don't always know if this task will be a leader or worker task when it's created, because if slices is set to "auto" it may
  * be either depending on the number of shards in the source indices. We figure that out when the request is handled and set it on this
  * class with {@link #setWorkerCount(int)} or {@link #setWorker(float, Integer)}.
diff --git a/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java b/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java
index aff9ec1f20e46..4963080f5916c 100644
--- a/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java
+++ b/server/src/main/java/org/opensearch/index/reindex/DeleteByQueryRequest.java
@@ -49,7 +49,7 @@
 /**
  * Creates a new {@link DeleteByQueryRequest} that uses scrolling and bulk requests to delete all documents matching
  * the query. This can have performance as well as visibility implications.
- *
+ * 
                        
  * Delete-by-query now has the following semantics:
  * 
                          
  *     
                        • it's {@code non-atomic}, a delete-by-query may fail at any time while some documents matching the query have already been
diff --git a/server/src/main/java/org/opensearch/index/search/MatchQuery.java b/server/src/main/java/org/opensearch/index/search/MatchQuery.java
index ade4fb1e69586..9e2b79971369d 100644
--- a/server/src/main/java/org/opensearch/index/search/MatchQuery.java
+++ b/server/src/main/java/org/opensearch/index/search/MatchQuery.java
@@ -753,9 +753,9 @@ private Query analyzeGraphBoolean(String field, TokenStream source, BooleanClaus
                 lastState = end;
                 final Query queryPos;
                 boolean usePrefix = isPrefix && end == -1;
-                /**
-                 * check if the GraphTokenStreamFiniteStrings graph is empty
-                 * return empty BooleanQuery result
+                /*
+                  check if the GraphTokenStreamFiniteStrings graph is empty
+                  return empty BooleanQuery result
                  */
                 Iterator graphIt = graph.getFiniteStrings();
                 if (!graphIt.hasNext()) {
diff --git a/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java b/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java
index 0a4c197898d3d..155866e20d007 100644
--- a/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java
+++ b/server/src/main/java/org/opensearch/index/seqno/LocalCheckpointTracker.java
@@ -147,7 +147,7 @@ public synchronized void markSeqNoAsPersisted(final long seqNo) {
 
     /**
      * Updates the processed sequence checkpoint to the given value.
-     *
+     * 
                          
      * This method is only used for segment replication since indexing doesn't
      * take place on the replica allowing us to avoid the check that all sequence numbers
      * are consecutively processed.
@@ -208,7 +208,7 @@ public long getMaxSeqNo() {
 
     /**
      * constructs a {@link SeqNoStats} object, using local state and the supplied global checkpoint
-     *
+     * 
                          
      * This is needed to make sure the persisted local checkpoint and max seq no are consistent
      */
     public synchronized SeqNoStats getStats(final long globalCheckpoint) {
diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
index 4b6d72b86ff62..94f376d923689 100644
--- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
@@ -88,7 +88,7 @@
 
 /**
  * This class is responsible for tracking the replication group with its progress and safety markers (local and global checkpoints).
- *
+ * 
                          
  * The global checkpoint is the highest sequence number for which all lower (or equal) sequence number have been processed
  * on all shards that are currently active. Since shards count as "active" when the cluster-manager starts
  * them, and before this primary shard has been notified of this fact, we also include shards that have completed recovery. These shards
@@ -112,10 +112,10 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
      *            checkpoint based on the local checkpoints of all in-sync shard copies.
      * - replica: this shard receives global checkpoint information from the primary (see
      *   {@link #updateGlobalCheckpointOnReplica(long, String)}).
-     *
+     * 
                          
      * When a shard is initialized (be it a primary or replica), it initially operates in replica mode. The global checkpoint tracker is
      * then switched to primary mode in the following three scenarios:
-     *
+     * 
                          
      * - An initializing primary shard that is not a relocation target is moved to primary mode (using {@link #activatePrimaryMode}) once
      *   the shard becomes active.
      * - An active replica shard is moved to primary mode (using {@link #activatePrimaryMode}) once it is promoted to primary.
@@ -140,7 +140,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
      * in-sync shard copies cannot grow, otherwise the relocation target might miss this information and increase the global checkpoint
      * to eagerly. As consequence, some of the methods in this class are not allowed to be called while a handoff is in progress,
      * in particular {@link #markAllocationIdAsInSync}.
-     *
+     * 
                          
      * A notable exception to this is the method {@link #updateFromClusterManager}, which is still allowed to be called during a relocation handoff.
      * The reason for this is that the handoff might fail and can be aborted (using {@link #abortRelocationHandoff}), in which case
      * it is important that the global checkpoint tracker does not miss any state updates that might happened during the handoff attempt.
@@ -1163,7 +1163,7 @@ public synchronized void updateGlobalCheckpointForShard(final String allocationI
 
     /**
      * Update the local knowledge of the visible checkpoint for the specified allocation ID.
-     *
+     * 
                          
      * This method will also stop timers for each shard and compute replication lag metrics.
      *
      * @param allocationId     the allocation ID to update the global checkpoint for
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java b/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java
index d1cb396f55d0f..3017c3ce6dcff 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexEventListener.java
@@ -155,7 +155,7 @@ default void beforeIndexShardDeleted(ShardId shardId, Settings indexSettings) {}
 
     /**
      * Called after the index shard has been deleted from disk.
-     *
+     * 
                          
      * Note: this method is only called if the deletion of the shard did finish without an exception
      *
      * @param shardId The shard id
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 2643af501bbc9..6ac75c7bfec17 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -1618,7 +1618,7 @@ public ReplicationCheckpoint getLatestReplicationCheckpoint() {
     /**
      * Compute and return the latest ReplicationCheckpoint for a shard and a GatedCloseable containing the corresponding SegmentInfos.
      * The segments referenced by the SegmentInfos will remain on disk until the GatedCloseable is closed.
-     *
+     * 
                          
      * Primary shards compute the seqNo used in the replication checkpoint from the fetched SegmentInfos.
      * Replica shards compute the seqNo from its latest processed checkpoint, which only increases when refreshing on new segments.
      *
@@ -3774,9 +3774,9 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro
             );
         }
 
-        /**
-         * With segment replication enabled for primary relocation, recover replica shard initially as read only and
-         * change to a writeable engine during relocation handoff after a round of segment replication.
+        /*
+          With segment replication enabled for primary relocation, recover replica shard initially as read only and
+          change to a writeable engine during relocation handoff after a round of segment replication.
          */
         boolean isReadOnlyReplica = indexSettings.isSegRepEnabled()
             && (shardRouting.primary() == false
@@ -4414,7 +4414,7 @@ public final boolean isSearchIdle() {
     /**
      *
      * Returns true if this shard supports search idle.
-     *
+     * 
                          
      * Indices using Segment Replication will ignore search idle unless there are no replicas.
      * Primary shards push out new segments only
      * after a refresh, so we don't want to wait for a search to trigger that cycle. Replicas will only refresh after receiving
diff --git a/server/src/main/java/org/opensearch/index/shard/RefreshListeners.java b/server/src/main/java/org/opensearch/index/shard/RefreshListeners.java
index 7dbbcbb2d7d20..803db773efe6c 100644
--- a/server/src/main/java/org/opensearch/index/shard/RefreshListeners.java
+++ b/server/src/main/java/org/opensearch/index/shard/RefreshListeners.java
@@ -54,7 +54,7 @@
 /**
  * Allows for the registration of listeners that are called when a change becomes visible for search. This functionality is exposed from
  * {@link IndexShard} but kept here so it can be tested without standing up the entire thing.
- *
+ * 
                          
  * When {@link Closeable#close()}d it will no longer accept listeners and flush any existing listeners.
  *
  * @opensearch.internal
@@ -86,7 +86,7 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener,
      * List of refresh listeners. Defaults to null and built on demand because most refresh cycles won't need it. Entries are never removed
      * from it, rather, it is nulled and rebuilt when needed again. The (hopefully) rare entries that didn't make the current refresh cycle
      * are just added back to the new list. Both the reference and the contents are always modified while synchronized on {@code this}.
-     *
+     * 
                          
      * We never set this to non-null while closed it {@code true}.
      */
     private volatile List>> refreshListeners = null;
diff --git a/server/src/main/java/org/opensearch/index/similarity/SimilarityProvider.java b/server/src/main/java/org/opensearch/index/similarity/SimilarityProvider.java
index ad64f3a55228f..085e93c794fb7 100644
--- a/server/src/main/java/org/opensearch/index/similarity/SimilarityProvider.java
+++ b/server/src/main/java/org/opensearch/index/similarity/SimilarityProvider.java
@@ -70,23 +70,23 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         SimilarityProvider that = (SimilarityProvider) o;
-        /**
-         * We check name only because the similarity is
-         * re-created for each new instance and they don't implement equals.
-         * This is not entirely correct though but we only use equality checks
-         * for similarities inside the same index and names are unique in this case.
-         **/
+        /*
+          We check name only because the similarity is
+          re-created for each new instance and they don't implement equals.
+          This is not entirely correct though but we only use equality checks
+          for similarities inside the same index and names are unique in this case.
+         */
         return Objects.equals(name, that.name);
     }
 
     @Override
     public int hashCode() {
-        /**
-         * We use name only because the similarity is
-         * re-created for each new instance and they don't implement equals.
-         * This is not entirely correct though but we only use equality checks
-         * for similarities a single index and names are unique in this case.
-         **/
+        /*
+          We use name only because the similarity is
+          re-created for each new instance and they don't implement equals.
+          This is not entirely correct though but we only use equality checks
+          for similarities a single index and names are unique in this case.
+         */
         return Objects.hash(name);
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java
index eefc1469a06a0..aa5d90cc65803 100644
--- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java
+++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java
@@ -322,10 +322,10 @@ public String snapshot() {
         return snapshot;
     }
 
-    /**
-     * Returns list of files in the shard
-     *
-     * @return list of files
+    /*
+      Returns list of files in the shard
+
+      @return list of files
      */
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java
index 9f505121d8dfa..b822742de6e97 100644
--- a/server/src/main/java/org/opensearch/index/store/Store.java
+++ b/server/src/main/java/org/opensearch/index/store/Store.java
@@ -299,14 +299,15 @@ final void ensureOpen() {
     /**
      * Returns a new MetadataSnapshot for the given commit. If the given commit is null
      * the latest commit point is used.
-     *
+     * 
                          
      * Note that this method requires the caller verify it has the right to access the store and
      * no concurrent file changes are happening. If in doubt, you probably want to use one of the following:
-     *
+     * 
                          
      * {@link #readMetadataSnapshot(Path, ShardId, NodeEnvironment.ShardLocker, Logger)} to read a meta data while locking
      * {@link IndexShard#snapshotStoreMetadata()} to safely read from an existing shard
      * {@link IndexShard#acquireLastIndexCommit(boolean)} to get an {@link IndexCommit} which is safe to use but has to be freed
-     * @param commit the index commit to read the snapshot from or null if the latest snapshot should be read from the
+     *
+     * @param commit the index commit to read the snapshot from or {@code null} if the latest snapshot should be read from the
      *               directory
      * @throws CorruptIndexException      if the lucene index is corrupted. This can be caused by a checksum mismatch or an
      *                                    unexpected exception when opening the index reading the segments file.
@@ -330,10 +331,10 @@ public MetadataSnapshot getMetadata() throws IOException {
     /**
      * Returns a new MetadataSnapshot for the given commit. If the given commit is null
      * the latest commit point is used.
-     *
+     * 
                          
      * Note that this method requires the caller verify it has the right to access the store and
      * no concurrent file changes are happening. If in doubt, you probably want to use one of the following:
-     *
+     * 
                          
      * {@link #readMetadataSnapshot(Path, ShardId, NodeEnvironment.ShardLocker, Logger)} to read a meta data while locking
      * {@link IndexShard#snapshotStoreMetadata()} to safely read from an existing shard
      * {@link IndexShard#acquireLastIndexCommit(boolean)} to get an {@link IndexCommit} which is safe to use but has to be freed
@@ -806,7 +807,7 @@ public void cleanupAndVerify(String reason, MetadataSnapshot sourceMetadata) thr
 
     /**
      * Segment replication method
-     *
+     * 
                          
      * This method takes the segment info bytes to build SegmentInfos. It inc'refs files pointed by passed in SegmentInfos
      * bytes to ensure they are not deleted.
      *
@@ -881,7 +882,7 @@ public void beforeClose() {
      * have the ability to create a writer directly from a SegmentInfos object.  To promote the replica as a primary and avoid reindexing, we must first commit
      * on the replica so that it can be opened with a writeable engine. Further, InternalEngine currently invokes `trimUnsafeCommits` which reverts the engine to a previous safeCommit where the max seqNo is less than or equal
      * to the current global checkpoint. It is likely that the replica has a maxSeqNo that is higher than the global cp and a new commit will be wiped.
-     *
+     * 
                          
      * To get around these limitations, this method first creates an IndexCommit directly from SegmentInfos, it then
      * uses an appending IW to create an IndexCommit from the commit created on SegmentInfos.
      * This ensures that 1. All files in the new commit are fsynced and 2. Deletes older commit points so the only commit to start from is our new commit.
diff --git a/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockIndexInput.java b/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockIndexInput.java
index 7319a5324777a..6fd198747570f 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockIndexInput.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/file/OnDemandBlockIndexInput.java
@@ -27,7 +27,7 @@
  * 
                          
  * This class delegate the responsibility of actually fetching the block when demanded to its subclasses using
  * {@link OnDemandBlockIndexInput#fetchBlock(int)}.
- *
+ * 
                          
  * Like {@link IndexInput}, this class may only be used from one thread as it is not thread safe.
  * However, a cleaning action may run from another thread triggered by the {@link Cleaner}, but
  * this is okay because at that point the {@link OnDemandBlockIndexInput} instance is phantom
@@ -428,10 +428,10 @@ Builder blockSizeShift(int blockSizeShift) {
      * instance to hold the current underlying IndexInput, while allowing it to
      * be changed out with different instances as {@link OnDemandBlockIndexInput}
      * reads through the data.
-     *
+     * 
                          
      * This class implements {@link Runnable} so that it can be passed directly
      * to the cleaner to run its close action.
-     *
+     * 
                          
      * [1]: https://github.com/apache/lucene/blob/8340b01c3cc229f33584ce2178b07b8984daa6a9/lucene/core/src/java/org/apache/lucene/store/IndexInput.java#L32-L33
      */
     private static class BlockHolder implements Closeable, Runnable {
diff --git a/server/src/main/java/org/opensearch/index/translog/DefaultTranslogDeletionPolicy.java b/server/src/main/java/org/opensearch/index/translog/DefaultTranslogDeletionPolicy.java
index efc762ef00d52..05049e5d07373 100644
--- a/server/src/main/java/org/opensearch/index/translog/DefaultTranslogDeletionPolicy.java
+++ b/server/src/main/java/org/opensearch/index/translog/DefaultTranslogDeletionPolicy.java
@@ -14,7 +14,7 @@
 /**
  * Default implementation for the {@link TranslogDeletionPolicy}. Plugins can override the default behaviour
  * via the {@link org.opensearch.plugins.EnginePlugin#getCustomTranslogDeletionPolicyFactory()}.
- *
+ * 
                          
  * The default policy uses total number, size in bytes and maximum age for files.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java
index baa3737d576de..cf7f18840a03e 100644
--- a/server/src/main/java/org/opensearch/index/translog/Translog.java
+++ b/server/src/main/java/org/opensearch/index/translog/Translog.java
@@ -840,7 +840,7 @@ public boolean ensureSynced(Stream locations) throws IOException {
 
     /**
      * Closes the translog if the current translog writer experienced a tragic exception.
-     *
+     * 
                          
      * Note that in case this thread closes the translog it must not already be holding a read lock on the translog as it will acquire a
      * write lock in the course of closing the translog
      *
@@ -1976,7 +1976,7 @@ static String createEmptyTranslog(
     /**
      * Creates a new empty translog within the specified {@code location} that contains the given {@code initialGlobalCheckpoint},
      * {@code primaryTerm} and {@code translogUUID}.
-     *
+     * 
                          
      * This method should be used directly under specific circumstances like for shards that will see no indexing. Specifying a non-unique
      * translog UUID could cause a lot of issues and that's why in all (but one) cases the method
      * {@link #createEmptyTranslog(Path, long, ShardId, long)} should be used instead.
diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java
index a5f0607431a8b..3f33c155be15e 100644
--- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java
+++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java
@@ -376,7 +376,7 @@ synchronized boolean assertNoSeqAbove(long belowTerm, long aboveSeqNo) {
 
     /**
      * write all buffered ops to disk and fsync file.
-     *
+     * 
                          
      * Note: any exception during the sync process will be interpreted as a tragic exception and the writer will be closed before
      * raising the exception.
      * @return  true if this call caused an actual sync operation
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index e27a597007b62..50c551c2be29b 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -926,7 +926,7 @@ private EngineFactory getEngineFactory(final IndexSettings idxSettings) {
     /**
      * creates a new mapper service for the given index, in order to do administrative work like mapping updates.
      * This *should not* be used for document parsing. Doing so will result in an exception.
-     *
+     * 
                          
      * Note: the returned {@link MapperService} should be closed when unneeded.
      */
     public synchronized MapperService createIndexMapperService(IndexMetadata indexMetadata) throws IOException {
@@ -1138,7 +1138,7 @@ public void deleteUnassignedIndex(String reason, IndexMetadata metadata, Cluster
     /**
      * Deletes the index store trying to acquire all shards locks for this index.
      * This method will delete the metadata for the index even if the actual shards can't be locked.
-     *
+     * 
                          
      * Package private for testing
      */
     void deleteIndexStore(String reason, IndexMetadata metadata) throws IOException {
@@ -1219,7 +1219,7 @@ public void deleteShardStore(String reason, ShardLock lock, IndexSettings indexS
      * This method deletes the shard contents on disk for the given shard ID. This method will fail if the shard deleting
      * is prevented by {@link #canDeleteShardContent(ShardId, IndexSettings)}
      * of if the shards lock can not be acquired.
-     *
+     * 
                          
      * On data nodes, if the deleted shard is the last shard folder in its index, the method will attempt to remove
      * the index folder as well.
      *
diff --git a/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java b/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java
index d9a64781c3f46..e345b613eebbd 100644
--- a/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java
+++ b/server/src/main/java/org/opensearch/indices/ShardLimitValidator.java
@@ -54,7 +54,7 @@
 /**
  * This class contains the logic used to check the cluster-wide shard limit before shards are created and ensuring that the limit is
  * updated correctly on setting updates, etc.
- *
+ * 
                          
  * NOTE: This is the limit applied at *shard creation time*. If you are looking for the limit applied at *allocation* time, which is
  * controlled by a different setting,
  * see {@link org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider}.
diff --git a/server/src/main/java/org/opensearch/indices/analysis/PreBuiltCacheFactory.java b/server/src/main/java/org/opensearch/indices/analysis/PreBuiltCacheFactory.java
index 601bd79a24746..13cc78b620b9a 100644
--- a/server/src/main/java/org/opensearch/indices/analysis/PreBuiltCacheFactory.java
+++ b/server/src/main/java/org/opensearch/indices/analysis/PreBuiltCacheFactory.java
@@ -48,11 +48,11 @@ public class PreBuiltCacheFactory {
 
     /**
      * The strategy of caching the analyzer
-     *
-     * ONE               Exactly one version is stored. Useful for analyzers which do not store version information
-     * LUCENE            Exactly one version for each lucene version is stored. Useful to prevent different analyzers with the same version
-     * OPENSEARCH        Exactly one version per opensearch version is stored. Useful if you change an analyzer between opensearch
-     *                   releases, when the lucene version does not change
+     * 
                            
+     * 
                          • ONE        : Exactly one version is stored. Useful for analyzers which do not store version information
                            • 
+     * 
                          • LUCENE     : Exactly one version for each lucene version is stored. Useful to prevent different analyzers with the same version
                            • 
+     * 
                          • OPENSEARCH : Exactly one version per opensearch version is stored. Useful if you change an analyzer between opensearch releases, when the lucene version does not change
                            • 
+     * 
                          
      */
     public enum CachingStrategy {
         ONE,
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTargetHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTargetHandler.java
index ac28dabf815a5..707e41c8c27e1 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTargetHandler.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTargetHandler.java
@@ -55,7 +55,7 @@ public interface RecoveryTargetHandler extends FileChunkWriter {
 
     /**
      * Used with Segment replication only
-     *
+     * 
                          
      * This function is used to force a sync target primary node with source (old primary). This is to avoid segment files
      * conflict with replicas when target is promoted as primary.
      */
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
index 36beabc4a9026..66f5b13449f05 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
@@ -189,7 +189,7 @@ public void indexTranslogOperations(
 
     /**
      * Used with Segment replication only
-     *
+     * 
                          
      * This function is used to force a sync target primary node with source (old primary). This is to avoid segment files
      * conflict with replicas when target is promoted as primary.
      */
diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
index 268894b1c0af3..33967c0203516 100644
--- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
+++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
@@ -61,8 +61,8 @@ class OngoingSegmentReplications {
         this.allocationIdToHandlers = ConcurrentCollections.newConcurrentMap();
     }
 
-    /**
-     * Operations on the {@link #copyStateMap} member.
+    /*
+      Operations on the {@link #copyStateMap} member.
      */
 
     /**
@@ -85,12 +85,12 @@ synchronized CopyState getCachedCopyState(ReplicationCheckpoint checkpoint) thro
             // build the CopyState object and cache it before returning
             final CopyState copyState = new CopyState(checkpoint, indexShard);
 
-            /**
-             * Use the checkpoint from the request as the key in the map, rather than
-             * the checkpoint from the created CopyState. This maximizes cache hits
-             * if replication targets make a request with an older checkpoint.
-             * Replication targets are expected to fetch the checkpoint in the response
-             * CopyState to bring themselves up to date.
+            /*
+              Use the checkpoint from the request as the key in the map, rather than
+              the checkpoint from the created CopyState. This maximizes cache hits
+              if replication targets make a request with an older checkpoint.
+              Replication targets are expected to fetch the checkpoint in the response
+              CopyState to bring themselves up to date.
              */
             addToCopyStateMap(checkpoint, copyState);
             return copyState;
diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationRequestTracker.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationRequestTracker.java
index 39649727e31d7..3775be7b6da15 100644
--- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationRequestTracker.java
+++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationRequestTracker.java
@@ -58,8 +58,7 @@ public class ReplicationRequestTracker {
      * This method will mark that a request with a unique sequence number has been received. If this is the
      * first time the unique request has been received, this method will return a listener to be completed.
      * The caller should then perform the requested action and complete the returned listener.
-     *
-     *
+     * 
                          
      * If the unique request has already been received, this method will either complete the provided listener
      * or attach that listener to the listener returned in the first call. In this case, the method will
      * return null and the caller should not perform the requested action as a prior caller is already
diff --git a/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java
index f97388a8262ff..5185b740d90cb 100644
--- a/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java
+++ b/server/src/main/java/org/opensearch/ingest/ConfigurationUtils.java
@@ -74,7 +74,7 @@ private ConfigurationUtils() {}
 
     /**
      * Returns and removes the specified optional property from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type string a {@link OpenSearchParseException} is thrown.
      */
     public static String readOptionalStringProperty(
@@ -89,7 +89,7 @@ public static String readOptionalStringProperty(
 
     /**
      * Returns and removes the specified property from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type string an {@link OpenSearchParseException} is thrown.
      * If the property is missing an {@link OpenSearchParseException} is thrown
      */
@@ -104,7 +104,7 @@ public static String readStringProperty(
 
     /**
      * Returns and removes the specified property from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type string a {@link OpenSearchParseException} is thrown.
      * If the property is missing and no default value has been specified a {@link OpenSearchParseException} is thrown
      */
@@ -141,7 +141,7 @@ private static String readString(String processorType, String processorTag, Stri
 
     /**
      * Returns and removes the specified property from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type string or int a {@link OpenSearchParseException} is thrown.
      * If the property is missing and no default value has been specified a {@link OpenSearchParseException} is thrown
      */
@@ -180,7 +180,7 @@ private static String readStringOrInt(String processorType, String processorTag,
 
     /**
      * Returns and removes the specified property from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type string or int a {@link OpenSearchParseException} is thrown.
      */
     public static String readOptionalStringOrIntProperty(
@@ -228,7 +228,7 @@ private static Boolean readBoolean(String processorType, String processorTag, St
 
     /**
      * Returns and removes the specified property from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type int a {@link OpenSearchParseException} is thrown.
      * If the property is missing an {@link OpenSearchParseException} is thrown
      */
@@ -257,7 +257,7 @@ public static Integer readIntProperty(
 
     /**
      * Returns and removes the specified property from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type int a {@link OpenSearchParseException} is thrown.
      * If the property is missing an {@link OpenSearchParseException} is thrown
      */
@@ -285,7 +285,7 @@ public static Double readDoubleProperty(
 
     /**
      * Returns and removes the specified property of type list from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type list an {@link OpenSearchParseException} is thrown.
      */
     public static  List readOptionalList(
@@ -303,7 +303,7 @@ public static  List readOptionalList(
 
     /**
      * Returns and removes the specified property of type list from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type list an {@link OpenSearchParseException} is thrown.
      * If the property is missing an {@link OpenSearchParseException} is thrown
      */
@@ -333,7 +333,7 @@ private static  List readList(String processorType, String processorTag, S
 
     /**
      * Returns and removes the specified property of type map from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type map an {@link OpenSearchParseException} is thrown.
      * If the property is missing an {@link OpenSearchParseException} is thrown
      */
@@ -353,7 +353,7 @@ public static  Map readMap(
 
     /**
      * Returns and removes the specified property of type map from the specified configuration map.
-     *
+     * 
                          
      * If the property value isn't of type map an {@link OpenSearchParseException} is thrown.
      */
     public static  Map readOptionalMap(
diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java
index 91003e963e302..baf357a4bc0d5 100644
--- a/server/src/main/java/org/opensearch/ingest/IngestService.java
+++ b/server/src/main/java/org/opensearch/ingest/IngestService.java
@@ -695,7 +695,7 @@ public IngestStats stats() {
     /**
      * Adds a listener that gets invoked with the current cluster state before processor factories
      * get invoked.
-     *
+     * 
                          
      * This is useful for components that are used by ingest processors, so that they have the opportunity to update
      * before these components get used by the ingest processor factory.
      */
diff --git a/server/src/main/java/org/opensearch/ingest/Pipeline.java b/server/src/main/java/org/opensearch/ingest/Pipeline.java
index 37643e48ed2be..2541cfbf4af77 100644
--- a/server/src/main/java/org/opensearch/ingest/Pipeline.java
+++ b/server/src/main/java/org/opensearch/ingest/Pipeline.java
@@ -123,8 +123,8 @@ public static Pipeline create(
 
     /**
      * Modifies the data of a document to be indexed based on the processor this pipeline holds
-     *
-     * If null is returned then this document will be dropped and not indexed, otherwise
+     * 
                          
+     * If {@code null} is returned then this document will be dropped and not indexed, otherwise
      * this document will be kept and indexed.
      */
     public void execute(IngestDocument ingestDocument, BiConsumer handler) {
diff --git a/server/src/main/java/org/opensearch/ingest/Processor.java b/server/src/main/java/org/opensearch/ingest/Processor.java
index e0f196dfcb115..6097045a87e21 100644
--- a/server/src/main/java/org/opensearch/ingest/Processor.java
+++ b/server/src/main/java/org/opensearch/ingest/Processor.java
@@ -48,7 +48,7 @@
 /**
  * A processor implementation may modify the data belonging to a document.
  * Whether changes are made and what exactly is modified is up to the implementation.
- *
+ * 
                          
  * Processors may get called concurrently and thus need to be thread-safe.
  *
  * @opensearch.internal
@@ -57,7 +57,7 @@ public interface Processor {
 
     /**
      * Introspect and potentially modify the incoming data.
-     *
+     * 
                          
      * Expert method: only override this method if a processor implementation needs to make an asynchronous call,
      * otherwise just overwrite {@link #execute(IngestDocument)}.
      */
diff --git a/server/src/main/java/org/opensearch/ingest/ValueSource.java b/server/src/main/java/org/opensearch/ingest/ValueSource.java
index 0ef7c3373596d..3463fb0f83b26 100644
--- a/server/src/main/java/org/opensearch/ingest/ValueSource.java
+++ b/server/src/main/java/org/opensearch/ingest/ValueSource.java
@@ -56,7 +56,7 @@ public interface ValueSource {
 
     /**
      * Returns a copy of the value this ValueSource holds and resolves templates if there're any.
-     *
+     * 
                          
      * For immutable values only a copy of the reference to the value is made.
      *
      * @param model The model to be used when resolving any templates
diff --git a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
index 98229941252ba..a0a14372aa31a 100644
--- a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
+++ b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
@@ -59,13 +59,13 @@
 /**
  * The {@link OsProbe} class retrieves information about the physical and swap size of the machine
  * memory, as well as the system load average and cpu load.
- *
+ * 
                          
  * In some exceptional cases, it's possible the underlying native methods used by
  * {@link #getFreePhysicalMemorySize()}, {@link #getTotalPhysicalMemorySize()},
  * {@link #getFreeSwapSpaceSize()}, and {@link #getTotalSwapSpaceSize()} can return a
  * negative value. Because of this, we prevent those methods from returning negative values,
  * returning 0 instead.
- *
+ * 
                          
  * The OS can report a negative number in a number of cases:
  * - Non-supported OSes (HP-UX, or AIX)
  * - A failure of macOS to initialize host statistics
@@ -183,11 +183,11 @@ public long getTotalSwapSpaceSize() {
 
     /**
      * The system load averages as an array.
-     *
+     * 
                          
      * On Windows, this method returns {@code null}.
-     *
+     * 
                          
      * On Linux, this method returns the 1, 5, and 15-minute load averages.
-     *
+     * 
                          
      * On macOS, this method should return the 1-minute load average.
      *
      * @return the available system load averages or {@code null}
diff --git a/server/src/main/java/org/opensearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/opensearch/persistent/NodePersistentTasksExecutor.java
index 209df1e1f498d..403630b89e42a 100644
--- a/server/src/main/java/org/opensearch/persistent/NodePersistentTasksExecutor.java
+++ b/server/src/main/java/org/opensearch/persistent/NodePersistentTasksExecutor.java
@@ -37,7 +37,7 @@
 
 /**
  * This component is responsible for execution of persistent tasks.
- *
+ * 
                          
  * It abstracts away the execution of tasks and greatly simplifies testing of PersistentTasksNodeService
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/opensearch/persistent/PersistentTasksClusterService.java
index 89bb23930b063..4e38fb34dbf17 100644
--- a/server/src/main/java/org/opensearch/persistent/PersistentTasksClusterService.java
+++ b/server/src/main/java/org/opensearch/persistent/PersistentTasksClusterService.java
@@ -323,7 +323,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
 
     /**
      * This unassigns a task from any node, i.e. it is assigned to a {@code null} node with the provided reason.
-     *
+     * 
                          
      * Since the assignment executor node is null, the {@link PersistentTasksClusterService} will attempt to reassign it to a valid
      * node quickly.
      *
diff --git a/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java b/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java
index 7e7345811246f..3552c8286b7a3 100644
--- a/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java
@@ -46,9 +46,9 @@ public interface CircuitBreakerPlugin {
 
     /**
      * Each of the factory functions are passed to the configured {@link CircuitBreakerService}.
-     *
+     * 
                          
      * The service then constructs a {@link CircuitBreaker} given the resulting {@link BreakerSettings}.
-     *
+     * 
                          
      * Custom circuit breakers settings can be found in {@link BreakerSettings}.
      * See:
      *  - limit (example: `breaker.foo.limit`) {@link BreakerSettings#CIRCUIT_BREAKER_LIMIT_SETTING}
@@ -63,7 +63,7 @@ public interface CircuitBreakerPlugin {
     /**
      * The passed {@link CircuitBreaker} object is the same one that was constructed by the {@link BreakerSettings}
      * provided by {@link CircuitBreakerPlugin#getCircuitBreaker(Settings)}.
-     *
+     * 
                          
      * This reference should never change throughout the lifetime of the node.
      *
      * @param circuitBreaker The constructed {@link CircuitBreaker} object from the {@link BreakerSettings}
diff --git a/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java b/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java
index c2e147b86d17f..1edd9f52d97a7 100644
--- a/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/ClusterPlugin.java
@@ -64,7 +64,7 @@ default Collection createAllocationDeciders(Settings settings
 
     /**
      * Return {@link ShardsAllocator} implementations added by this plugin.
-     *
+     * 
                          
      * The key of the returned {@link Map} is the name of the allocator, and the value
      * is a function to construct the allocator.
      *
@@ -88,7 +88,7 @@ default Map getExistingShardsAllocators() {
     /**
      * Called when the node is started
      *
-     * DEPRECATED: Use {@link #onNodeStarted(DiscoveryNode)} for newer implementations.
+     * @deprecated Use {@link #onNodeStarted(DiscoveryNode)} for newer implementations.
      */
     @Deprecated
     default void onNodeStarted() {}
diff --git a/server/src/main/java/org/opensearch/plugins/DiscoveryPlugin.java b/server/src/main/java/org/opensearch/plugins/DiscoveryPlugin.java
index bca72942bd70e..63f0d826b592f 100644
--- a/server/src/main/java/org/opensearch/plugins/DiscoveryPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/DiscoveryPlugin.java
@@ -68,13 +68,10 @@ public interface DiscoveryPlugin {
      * This can be handy if you want to provide your own Network interface name like _mycard_
      * and implement by yourself the logic to get an actual IP address/hostname based on this
      * name.
-     *
+     * 
                          
      * For example: you could call a third party service (an API) to resolve _mycard_.
      * Then you could define in opensearch.yml settings like:
-     *
-     * 
                          {@code
-     * network.host: _mycard_
-     * }
                          
+     * {@code network.host: _mycard_ }
      */
     default NetworkService.CustomNameResolver getCustomNameResolver(Settings settings) {
         return null;
@@ -82,7 +79,7 @@ default NetworkService.CustomNameResolver getCustomNameResolver(Settings setting
 
     /**
      * Returns providers of seed hosts for discovery.
-     *
+     * 
                          
      * The key of the returned map is the name of the host provider
      * (see {@link org.opensearch.discovery.DiscoveryModule#DISCOVERY_SEED_PROVIDERS_SETTING}), and
      * the value is a supplier to construct the host provider when it is selected for use.
diff --git a/server/src/main/java/org/opensearch/plugins/EnginePlugin.java b/server/src/main/java/org/opensearch/plugins/EnginePlugin.java
index 92ae2ff9cd661..5ea5442d84ffa 100644
--- a/server/src/main/java/org/opensearch/plugins/EnginePlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/EnginePlugin.java
@@ -92,7 +92,7 @@ default Optional getCustomCodecServiceFactory(IndexSettings
      * When an index is created this method is invoked for each engine plugin. Engine plugins that need to provide a
      * custom {@link TranslogDeletionPolicy} can override this method to return a function that takes the {@link IndexSettings}
      * and a {@link Supplier} for {@link RetentionLeases} and returns a custom {@link TranslogDeletionPolicy}.
-     *
+     * 
                          
      * Only one of the installed Engine plugins can override this otherwise {@link IllegalStateException} will be thrown.
      *
      * @return a function that returns an instance of {@link TranslogDeletionPolicy}
diff --git a/server/src/main/java/org/opensearch/plugins/ExtensiblePlugin.java b/server/src/main/java/org/opensearch/plugins/ExtensiblePlugin.java
index 4dd4010383934..367d335ac4fea 100644
--- a/server/src/main/java/org/opensearch/plugins/ExtensiblePlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/ExtensiblePlugin.java
@@ -36,7 +36,7 @@
 
 /**
  * An extension point for {@link Plugin} implementations to be themselves extensible.
- *
+ * 
                          
  * This class provides a callback for extensible plugins to be informed of other plugins
  * which extend them.
  *
@@ -62,7 +62,7 @@ interface ExtensionLoader {
 
     /**
      * Allow this plugin to load extensions from other plugins.
-     *
+     * 
                          
      * This method is called once only, after initializing this plugin and all plugins extending this plugin. It is called before
      * any other methods on this Plugin instance are called.
      */
diff --git a/server/src/main/java/org/opensearch/plugins/IdentityPlugin.java b/server/src/main/java/org/opensearch/plugins/IdentityPlugin.java
index 00f3f8aff585c..410535504f0dd 100644
--- a/server/src/main/java/org/opensearch/plugins/IdentityPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/IdentityPlugin.java
@@ -19,16 +19,14 @@
 public interface IdentityPlugin {
 
     /**
-     * Get the current subject
-     *
-     * Should never return null
+     * Get the current subject.
+     * @return Should never return null
      * */
     public Subject getSubject();
 
     /**
      * Get the Identity Plugin's token manager implementation
-     *
-     * Should never return null
+     * @return Should never return null.
      */
     public TokenManager getTokenManager();
 }
diff --git a/server/src/main/java/org/opensearch/plugins/IngestPlugin.java b/server/src/main/java/org/opensearch/plugins/IngestPlugin.java
index dfc11aa8eea55..dc4f22de71344 100644
--- a/server/src/main/java/org/opensearch/plugins/IngestPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/IngestPlugin.java
@@ -46,7 +46,7 @@ public interface IngestPlugin {
 
     /**
      * Returns additional ingest processor types added by this plugin.
-     *
+     * 
                          
      * The key of the returned {@link Map} is the unique name for the processor which is specified
      * in pipeline configurations, and the value is a {@link org.opensearch.ingest.Processor.Factory}
      * to create the processor from a given pipeline configuration.
diff --git a/server/src/main/java/org/opensearch/plugins/Plugin.java b/server/src/main/java/org/opensearch/plugins/Plugin.java
index 0743cd3807eff..48486a6b55dfd 100644
--- a/server/src/main/java/org/opensearch/plugins/Plugin.java
+++ b/server/src/main/java/org/opensearch/plugins/Plugin.java
@@ -121,7 +121,7 @@ public Collection> getGuiceServiceClasses()
 
     /**
      * Returns components added by this plugin.
-     *
+     * 
                          
      * Any components returned that implement {@link LifecycleComponent} will have their lifecycle managed.
      * Note: To aid in the migration away from guice, all objects returned as components will be bound in guice
      * to themselves.
diff --git a/server/src/main/java/org/opensearch/plugins/PluginsService.java b/server/src/main/java/org/opensearch/plugins/PluginsService.java
index 47195a0264750..cc9cc5b5b5fbf 100644
--- a/server/src/main/java/org/opensearch/plugins/PluginsService.java
+++ b/server/src/main/java/org/opensearch/plugins/PluginsService.java
@@ -468,7 +468,7 @@ private static Bundle readPluginBundle(final Set bundles, final Path plu
 
     /**
      * Return the given bundles, sorted in dependency loading order.
-     *
+     * 
                          
      * This sort is stable, so that if two plugins do not have any interdependency,
      * their relative order from iteration of the provided set will not change.
      *
diff --git a/server/src/main/java/org/opensearch/plugins/RepositoryPlugin.java b/server/src/main/java/org/opensearch/plugins/RepositoryPlugin.java
index 189ba3cfc16ab..09233d49f3aea 100644
--- a/server/src/main/java/org/opensearch/plugins/RepositoryPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/RepositoryPlugin.java
@@ -52,9 +52,8 @@ public interface RepositoryPlugin {
      * Returns repository types added by this plugin.
      *
      * @param env The environment for the local node, which may be used for the local settings and path.repo
-     *
-     * The key of the returned {@link Map} is the type name of the repository and
-     * the value is a factory to construct the {@link Repository} interface.
+     *            The key of the returned {@link Map} is the type name of the repository and
+     *            the value is a factory to construct the {@link Repository} interface.
      */
     default Map getRepositories(
         Environment env,
@@ -70,9 +69,8 @@ default Map getRepositories(
      * through the external API.
      *
      * @param env The environment for the local node, which may be used for the local settings and path.repo
-     *
-     * The key of the returned {@link Map} is the type name of the repository and
-     * the value is a factory to construct the {@link Repository} interface.
+     *            The key of the returned {@link Map} is the type name of the repository and
+     *            the value is a factory to construct the {@link Repository} interface.
      */
     default Map getInternalRepositories(
         Environment env,
diff --git a/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java
index d2ef2b65c5944..7288a8caaec58 100644
--- a/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/SearchPipelinePlugin.java
@@ -35,7 +35,7 @@
 public interface SearchPipelinePlugin {
     /**
      * Returns additional search pipeline request processor types added by this plugin.
-     *
+     * 
                          
      * The key of the returned {@link Map} is the unique name for the processor which is specified
      * in pipeline configurations, and the value is a {@link org.opensearch.search.pipeline.Processor.Factory}
      * to create the processor from a given pipeline configuration.
@@ -46,7 +46,7 @@ default Map> getRequestProcess
 
     /**
      * Returns additional search pipeline response processor types added by this plugin.
-     *
+     * 
                          
      * The key of the returned {@link Map} is the unique name for the processor which is specified
      * in pipeline configurations, and the value is a {@link org.opensearch.search.pipeline.Processor.Factory}
      * to create the processor from a given pipeline configuration.
@@ -57,7 +57,7 @@ default Map> getResponseProce
 
     /**
      * Returns additional search pipeline search phase results processor types added by this plugin.
-     *
+     * 
                          
      * The key of the returned {@link Map} is the unique name for the processor which is specified
      * in pipeline configurations, and the value is a {@link org.opensearch.search.pipeline.Processor.Factory}
      * to create the processor from a given pipeline configuration.
diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java
index 8b5920e1f4485..40b4f97cd1897 100644
--- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java
@@ -551,8 +551,8 @@ class CompositeAggregationSpec {
         private final Consumer aggregatorRegistrar;
         private final Class valueSourceBuilderClass;
         @Deprecated
-        /** This is added for backward compatibility, you don't need to set it, as we use aggregationType instead of
-         * byte code
+        /* This is added for backward compatibility, you don't need to set it, as we use aggregationType instead of
+          byte code
          */
         private Byte byteCode;
         private final CompositeAggregationParsingFunction parsingFunction;
diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java
index 0b0644ce932e5..d88eeb5aa9c49 100644
--- a/server/src/main/java/org/opensearch/repositories/IndexId.java
+++ b/server/src/main/java/org/opensearch/repositories/IndexId.java
@@ -80,7 +80,7 @@ public String getName() {
      * The unique ID for the index within the repository.  This is *not* the same as the
      * index's UUID, but merely a unique file/URL friendly identifier that a repository can
      * use to name blobs for the index.
-     *
+     * 
                          
      * We could not use the index's actual UUID (See {@link Index#getUUID()}) because in the
      * case of snapshot/restore, the index UUID in the snapshotted index will be different
      * from the index UUID assigned to it when it is restored. Hence, the actual index UUID
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index 8e6649c366eeb..69883e0d19c8d 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -2547,7 +2547,7 @@ private RepositoryMetadata getRepoMetadata(ClusterState state) {
      * the next version number from when the index blob was written.  Each individual index-N blob is
      * only written once and never overwritten.  The highest numbered index-N blob is the latest one
      * that contains the current snapshots in the repository.
-     *
+     * 
                          
      * Package private for testing
      */
     long latestIndexBlobId() throws IOException {
diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java
index ac30f999d0da7..cc48b59699a17 100644
--- a/server/src/main/java/org/opensearch/rest/RestController.java
+++ b/server/src/main/java/org/opensearch/rest/RestController.java
@@ -524,7 +524,7 @@ private void handleBadRequest(String uri, RestRequest.Method method, RestChannel
     /**
      * Attempts to extract auth token and login.
      *
-     * @returns false if there was an error and the request should not continue being dispatched
+     * @return false if there was an error and the request should not continue being dispatched
      * */
     private boolean handleAuthenticateUser(final RestRequest request, final RestChannel channel) {
         try {
diff --git a/server/src/main/java/org/opensearch/script/Script.java b/server/src/main/java/org/opensearch/script/Script.java
index a611e71c3bf3f..9e74314c281cd 100644
--- a/server/src/main/java/org/opensearch/script/Script.java
+++ b/server/src/main/java/org/opensearch/script/Script.java
@@ -70,9 +70,9 @@
  * {@link Script} represents used-defined input that can be used to
  * compile and execute a script from the {@link ScriptService}
  * based on the {@link ScriptType}.
- *
+ * 
                          
  * There are three types of scripts specified by {@link ScriptType}.
- *
+ * 
                          
  * The following describes the expected parameters for each type of script:
  *
  * 
                            
@@ -348,16 +348,16 @@ public static Script parse(Settings settings) {
 
     /**
      * This will parse XContent into a {@link Script}.  The following formats can be parsed:
-     *
+     * 
                            
      * The simple format defaults to an {@link ScriptType#INLINE} with no compiler options or user-defined params:
-     *
+     * 
                            
      * Example:
      * {@code
      * "return Math.log(doc.popularity) * 100;"
      * }
      *
      * The complex format where {@link ScriptType} and idOrCode are required while lang, options and params are not required.
-     *
+     * 
                            
      * {@code
      * {
      *     // Exactly one of "id" or "source" must be specified
@@ -391,7 +391,7 @@ public static Script parse(Settings settings) {
      *
      * This also handles templates in a special way.  If a complexly formatted query is specified as another complex
      * JSON object the query is assumed to be a template, and the format will be preserved.
-     *
+     * 
                            
      * {@code
      * {
      *     "source" : { "query" : ... },
@@ -605,7 +605,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     /**
      * This will build scripts into the following XContent structure:
-     *
+     * 
                            
      * {@code
      * {
      *     "<(id, source)>" : "",
@@ -635,10 +635,10 @@ public void writeTo(StreamOutput out) throws IOException {
      * }
      *
      * Note that lang, options, and params will only be included if there have been any specified.
-     *
+     * 
                            
      * This also handles templates in a special way.  If the {@link Script#CONTENT_TYPE_OPTION} option
      * is provided and the {@link ScriptType#INLINE} is specified then the template will be preserved as a raw field.
-     *
+     * 
                            
      * {@code
      * {
      *     "source" : { "query" : ... },
diff --git a/server/src/main/java/org/opensearch/script/ScriptCache.java b/server/src/main/java/org/opensearch/script/ScriptCache.java
index 3c50fc12dcacb..fb57e7cdfa5bd 100644
--- a/server/src/main/java/org/opensearch/script/ScriptCache.java
+++ b/server/src/main/java/org/opensearch/script/ScriptCache.java
@@ -158,7 +158,7 @@ public ScriptContextStats stats(String context) {
     /**
      * Check whether there have been too many compilations within the last minute, throwing a circuit breaking exception if so.
      * This is a variant of the token bucket algorithm: https://en.wikipedia.org/wiki/Token_bucket
-     *
+     * 
                            
      * It can be thought of as a bucket with water, every time the bucket is checked, water is added proportional to the amount of time that
      * elapsed since the last time it was checked. If there is enough water, some is removed and the request is allowed. If there is not
      * enough water the request is denied. Just like a normal bucket, if water is added that overflows the bucket, the extra water/capacity
diff --git a/server/src/main/java/org/opensearch/script/ScriptContext.java b/server/src/main/java/org/opensearch/script/ScriptContext.java
index 71ced303b062e..2180b6059dbef 100644
--- a/server/src/main/java/org/opensearch/script/ScriptContext.java
+++ b/server/src/main/java/org/opensearch/script/ScriptContext.java
@@ -40,7 +40,7 @@
 
 /**
  * The information necessary to compile and run a script.
- *
+ * 
                            
  * A {@link ScriptContext} contains the information related to a single use case and the interfaces
  * and methods necessary for a {@link ScriptEngine} to implement.
  * 
                            
diff --git a/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java b/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java
index ef64959f99a3c..30756ff702e8f 100644
--- a/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java
+++ b/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java
@@ -56,11 +56,11 @@
 /**
  * The allowable types, languages and their corresponding contexts.  When serialized there is a top level types_allowed list,
  * meant to reflect the setting script.allowed_types with the allowed types (eg inline, stored).
- *
+ * 
                            
  * The top-level language_contexts list of objects have the language (eg. painless,
  * mustache) and a list of contexts available for the language.  It is the responsibility of the caller to ensure
  * these contexts are filtered by the script.allowed_contexts setting.
- *
+ * 
                            
  * The json serialization of the object has the form:
  * 
  *     {
diff --git a/server/src/main/java/org/opensearch/script/ScriptMetadata.java b/server/src/main/java/org/opensearch/script/ScriptMetadata.java
index 5f529fccd213c..fd92d8f7f02db 100644
--- a/server/src/main/java/org/opensearch/script/ScriptMetadata.java
+++ b/server/src/main/java/org/opensearch/script/ScriptMetadata.java
@@ -183,9 +183,9 @@ static ScriptMetadata deleteStoredScript(ScriptMetadata previous, String id) {
 
     /**
      * This will parse XContent into {@link ScriptMetadata}.
-     *
+     * 
                            
      * The following format will be parsed:
-     *
+     * 
                            
      * {@code
      * {
      *     "" : "<{@link StoredScriptSource#fromXContent(XContentParser, boolean)}>",
@@ -356,7 +356,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     /**
      * This will write XContent from {@link ScriptMetadata}.  The following format will be written:
-     *
+     * 
                            
      * {@code
      * {
      *     "" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
diff --git a/server/src/main/java/org/opensearch/script/StoredScriptSource.java b/server/src/main/java/org/opensearch/script/StoredScriptSource.java
index 0ff44b3af890a..d1dae67d0e55f 100644
--- a/server/src/main/java/org/opensearch/script/StoredScriptSource.java
+++ b/server/src/main/java/org/opensearch/script/StoredScriptSource.java
@@ -308,7 +308,7 @@ public static StoredScriptSource parse(BytesReference content, MediaType mediaTy
 
     /**
      * This will parse XContent into a {@link StoredScriptSource}. The following format is what will be parsed:
-     *
+     * 
                            
      * {@code
      * {
      *     "script" : {
@@ -387,7 +387,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     /**
      * This will write XContent from a {@link StoredScriptSource}. The following format will be written:
-     *
+     * 
                            
      * {@code
      * {
      *     "script" : {
diff --git a/server/src/main/java/org/opensearch/search/MultiValueMode.java b/server/src/main/java/org/opensearch/search/MultiValueMode.java
index 0fbd41f062710..d812f1290efe3 100644
--- a/server/src/main/java/org/opensearch/search/MultiValueMode.java
+++ b/server/src/main/java/org/opensearch/search/MultiValueMode.java
@@ -542,7 +542,7 @@ public static MultiValueMode fromString(String sortMode) {
      * Return a {@link NumericDocValues} instance that can be used to sort documents
      * with this mode and the provided values. When a document has no value,
      * missingValue is returned.
-     *
+     * 
                            
      * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX
      */
     public NumericDocValues select(final SortedNumericDocValues values) {
@@ -583,12 +583,12 @@ protected long pick(SortedNumericDocValues values) throws IOException {
     /**
      * Return a {@link NumericDocValues} instance that can be used to sort root documents
      * with this mode, the provided values and filters for root/inner documents.
-     *
+     * 
                            
      * For every root document, the values of its inner documents will be aggregated.
      * If none of the inner documents has a value, then missingValue is returned.
-     *
+     * 
                            
      * Allowed Modes: SUM, AVG, MIN, MAX
-     *
+     * 
                            
      * NOTE: Calling the returned instance on docs that are not root docs is illegal
      *       The returned instance can only be evaluate the current and upcoming docs
      */
@@ -658,7 +658,7 @@ protected long pick(
      * Return a {@link NumericDoubleValues} instance that can be used to sort documents
      * with this mode and the provided values. When a document has no value,
      * missingValue is returned.
-     *
+     * 
                            
      * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX
      */
     public NumericDoubleValues select(final SortedNumericDoubleValues values) {
@@ -694,12 +694,12 @@ protected double pick(SortedNumericDoubleValues values) throws IOException {
     /**
      * Return a {@link NumericDoubleValues} instance that can be used to sort root documents
      * with this mode, the provided values and filters for root/inner documents.
-     *
+     * 
                            
      * For every root document, the values of its inner documents will be aggregated.
      * If none of the inner documents has a value, then missingValue is returned.
-     *
+     * 
                            
      * Allowed Modes: SUM, AVG, MIN, MAX
-     *
+     * 
                            
      * NOTE: Calling the returned instance on docs that are not root docs is illegal
      *       The returned instance can only be evaluate the current and upcoming docs
      */
@@ -761,7 +761,7 @@ protected double pick(
      * Return a {@link BinaryDocValues} instance that can be used to sort documents
      * with this mode and the provided values. When a document has no value,
      * missingValue is returned.
-     *
+     * 
                            
      * Allowed Modes: MIN, MAX
      */
     public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue) {
@@ -816,12 +816,12 @@ protected BytesRef pick(SortedBinaryDocValues values) throws IOException {
     /**
      * Return a {@link BinaryDocValues} instance that can be used to sort root documents
      * with this mode, the provided values and filters for root/inner documents.
-     *
+     * 
                            
      * For every root document, the values of its inner documents will be aggregated.
      * If none of the inner documents has a value, then missingValue is returned.
-     *
+     * 
                            
      * Allowed Modes: MIN, MAX
-     *
+     * 
                            
      * NOTE: Calling the returned instance on docs that are not root docs is illegal
      *       The returned instance can only be evaluate the current and upcoming docs
      */
@@ -889,7 +889,7 @@ protected BytesRef pick(
     /**
      * Return a {@link SortedDocValues} instance that can be used to sort documents
      * with this mode and the provided values.
-     *
+     * 
                            
      * Allowed Modes: MIN, MAX
      */
     public SortedDocValues select(final SortedSetDocValues values) {
@@ -949,11 +949,11 @@ protected int pick(SortedSetDocValues values) throws IOException {
     /**
      * Return a {@link SortedDocValues} instance that can be used to sort root documents
      * with this mode, the provided values and filters for root/inner documents.
-     *
+     * 
                            
      * For every root document, the values of its inner documents will be aggregated.
-     *
+     * 
                            
      * Allowed Modes: MIN, MAX
-     *
+     * 
                            
      * NOTE: Calling the returned instance on docs that are not root docs is illegal
      *       The returned instance can only be evaluate the current and upcoming docs
      */
diff --git a/server/src/main/java/org/opensearch/search/SearchExtBuilder.java b/server/src/main/java/org/opensearch/search/SearchExtBuilder.java
index 4d86c6c2e2277..1df58666f6fdb 100644
--- a/server/src/main/java/org/opensearch/search/SearchExtBuilder.java
+++ b/server/src/main/java/org/opensearch/search/SearchExtBuilder.java
@@ -47,10 +47,10 @@
  * Any state needs to be serialized as part of the {@link Writeable#writeTo(StreamOutput)} method and
  * read from the incoming stream, usually done adding a constructor that takes {@link StreamInput} as
  * an argument.
- *
+ * 
                            
  * Registration happens through {@link SearchPlugin#getSearchExts()}, which also needs a {@link CheckedFunction} that's able to parse
  * the incoming request from the REST layer into the proper {@link SearchExtBuilder} subclass.
- *
+ * 
                            
  * {@link #getWriteableName()} must return the same name as the one used for the registration
  * of the {@link SearchExtSpec}.
  *
diff --git a/server/src/main/java/org/opensearch/search/SearchHit.java b/server/src/main/java/org/opensearch/search/SearchHit.java
index 49f1cc585fd80..5649ec383a9e9 100644
--- a/server/src/main/java/org/opensearch/search/SearchHit.java
+++ b/server/src/main/java/org/opensearch/search/SearchHit.java
@@ -1035,7 +1035,7 @@ public int getOffset() {
 
         /**
          * Returns the next child nested level if there is any, otherwise null is returned.
-         *
+         * 
                            
          * In the case of mappings with multiple levels of nested object fields
          */
         public NestedIdentity getChild() {
diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java
index 0f1030b87c036..62d397de58187 100644
--- a/server/src/main/java/org/opensearch/search/SearchModule.java
+++ b/server/src/main/java/org/opensearch/search/SearchModule.java
@@ -327,7 +327,7 @@ public class SearchModule {
 
     /**
      * Constructs a new SearchModule object
-     *
+     * 
                            
      * NOTE: This constructor should not be called in production unless an accurate {@link Settings} object is provided.
      *       When constructed, a static flag is set in Lucene {@link BooleanQuery#setMaxClauseCount} according to the settings.
      * @param settings Current settings
diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java
index 392c65ce27aea..47e9def094623 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java
@@ -135,7 +135,7 @@ public ScoreMode scoreMode() {
      * Returns a converter for point values if it's safe to use the indexed data instead of
      * doc values.  Generally, this means that the query has no filters or scripts, the aggregation is
      * top level, and the underlying field is indexed, and the index is sorted in the right order.
-     *
+     * 
                            
      * If those conditions aren't met, return null to indicate a point reader cannot
      * be used in this case.
      *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java
index e4d64e1e8517c..288c7a64ba795 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/InternalAggregations.java
@@ -125,7 +125,7 @@ public double sortValue(AggregationPath.PathElement head, Iterator
      * This method first reduces the aggregations, and if it is the final reduce, then reduce the pipeline
      * aggregations (both embedded parent/sibling as well as top-level sibling pipelines)
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java
index 67af0b13eed3b..eef427754f535 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java
@@ -132,7 +132,7 @@ public final void collectExistingBucket(LeafBucketCollector subCollector, int do
     /**
      * This only tidies up doc counts. Call {@link MergingBucketsDeferringCollector#mergeBuckets(long[])}  to merge the actual
      * ordinals and doc ID deltas.
-     *
+     * 
                            
      * Refer to that method for documentation about the merge map.
      *
      * @deprecated use {@link mergeBuckets(long, LongUnaryOperator)}
@@ -146,7 +146,7 @@ public final void mergeBuckets(long[] mergeMap, long newNumBuckets) {
      *
      *  @param mergeMap a unary operator which maps a bucket's ordinal to the ordinal it should be merged with.
      *  If a bucket's ordinal is mapped to -1 then the bucket is removed entirely.
-     *
+     * 
                            
      * This only tidies up doc counts. Call {@link MergingBucketsDeferringCollector#mergeBuckets(LongUnaryOperator)} to
      * merge the actual ordinals and doc ID deltas.
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java
index de74055bb94f3..dfb8f6be7155d 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java
@@ -101,7 +101,7 @@ private GeoTileUtils() {}
 
     /**
      * Parse an integer precision (zoom level). The {@link ValueType#INT} allows it to be a number or a string.
-     *
+     * 
                            
      * The precision is expressed as a zoom level between 0 and {@link #MAX_ZOOM} (inclusive).
      *
      * @param parser {@link XContentParser} to parse the value from
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
index 8d35c1edc8cb0..3e9424eda92a9 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
@@ -55,16 +55,17 @@ public MergingBucketsDeferringCollector(SearchContext context, boolean isGlobal)
 
     /**
      * Merges/prunes the existing bucket ordinals and docDeltas according to the provided mergeMap.
-     *
+     * 
                            
      * The mergeMap is an array where the index position represents the current bucket ordinal, and
      * the value at that position represents the ordinal the bucket should be merged with.  If
      * the value is set to -1 it is removed entirely.
-     *
+     * 
                            
      * For example, if the mergeMap [1,1,3,-1,3] is provided:
-     *  - Buckets `0` and `1` will be merged to bucket ordinal `1`
-     *  - Bucket `2` and `4` will be merged to ordinal `3`
-     *  - Bucket `3` will be removed entirely
-     *
+     * 
                              
+     *  
                            •  Buckets `0` and `1` will be merged to bucket ordinal `1`
                              • 
+     *  
                            •  Bucket `2` and `4` will be merged to ordinal `3`
                              • 
+     *  
                            •  Bucket `3` will be removed entirely
                              • 
+     * 
                            
      *  This process rebuilds the ordinals and docDeltas according to the mergeMap, so it should
      *  not be called unless there are actually changes to be made, to avoid unnecessary work.
      *
@@ -80,7 +81,7 @@ public void mergeBuckets(long[] mergeMap) {
      *
      * @param mergeMap a unary operator which maps a bucket's ordinal to the ordinal it should be merged with.
      * If a bucket's ordinal is mapped to -1 then the bucket is removed entirely.
-     *
+     * 
                            
      * This process rebuilds the ordinals and docDeltas according to the mergeMap, so it should
      * not be called unless there are actually changes to be made, to avoid unnecessary work.
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
index 366de8619fd55..fd94ba355238a 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
@@ -216,7 +216,7 @@ public DateHistogramValuesSourceBuilder dateHistogramInterval(DateHistogramInter
     /**
      * Sets the interval of the DateHistogram using calendar units (`1d`, `1w`, `1M`, etc).  These units
      * are calendar-aware, meaning they respect leap additions, variable days per month, etc.
-     *
+     * 
                            
      * This is mutually exclusive with {@link DateHistogramValuesSourceBuilder#fixedInterval(DateHistogramInterval)}
      *
      * @param interval The calendar interval to use with the aggregation
@@ -229,7 +229,7 @@ public DateHistogramValuesSourceBuilder calendarInterval(DateHistogramInterval i
     /**
      * Sets the interval of the DateHistogram using fixed units (`1ms`, `1s`, `10m`, `4h`, etc).  These are
      * not calendar aware and are simply multiples of fixed, SI units.
-     *
+     * 
                            
      * This is mutually exclusive with {@link DateHistogramValuesSourceBuilder#calendarInterval(DateHistogramInterval)}
      *
      * @param interval The fixed interval to use with the aggregation
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java
index 27619044d8995..9f8a4cff5f3fc 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/InternalComposite.java
@@ -142,10 +142,10 @@ public String getWriteableName() {
 
     @Override
     public InternalComposite create(List newBuckets) {
-        /**
-         * This is used by pipeline aggregations to filter/remove buckets so we
-         * keep the afterKey of the original aggregation in order
-         * to be able to retrieve the next page even if all buckets have been filtered.
+        /*
+          This is used by pipeline aggregations to filter/remove buckets so we
+          keep the afterKey of the original aggregation in order
+          to be able to retrieve the next page even if all buckets have been filtered.
          */
         return new InternalComposite(
             name,
@@ -473,8 +473,8 @@ public int compareKey(InternalBucket other) {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            /**
-             * See {@link CompositeAggregation#bucketToXContent}
+            /*
+              See {@link CompositeAggregation#bucketToXContent}
              */
             throw new UnsupportedOperationException("not implemented");
         }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/ParsedComposite.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/ParsedComposite.java
index 8382b191025fe..2a1544e218f2c 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/ParsedComposite.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/ParsedComposite.java
@@ -70,9 +70,9 @@ public static ParsedComposite fromXContent(XContentParser parser, String name) t
         ParsedComposite aggregation = PARSER.parse(parser, null);
         aggregation.setName(name);
         if (aggregation.afterKey == null && aggregation.getBuckets().size() > 0) {
-            /**
-             * Previous versions (< 6.3) don't send afterKey
-             * in the response so we set it as the last returned buckets.
+            /*
+              Previous versions (< 6.3) don't send afterKey
+              in the response so we set it as the last returned buckets.
              */
             aggregation.setAfterKey(aggregation.getBuckets().get(aggregation.getBuckets().size() - 1).key);
         }
@@ -130,8 +130,8 @@ void setKey(Map key) {
 
         @Override
         public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            /**
-             * See {@link CompositeAggregation#bucketToXContent}
+            /*
+              See {@link CompositeAggregation#bucketToXContent}
              */
             throw new UnsupportedOperationException("not implemented");
         }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java
index bd0a4f13ddf08..9442529bf9342 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java
@@ -62,7 +62,7 @@ abstract class SortedDocsProducer {
      * Visits all non-deleted documents in iterator and fills the provided queue
      * with the top composite buckets extracted from the collection.
      * Documents that contain a top composite bucket are added in the provided builder if it is not null.
-     *
+     * 
                            
      * Returns true if the queue is full and the current leadSourceBucket did not produce any competitive
      * composite buckets.
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
index 55c841f5b9c04..a0a636c121e12 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
@@ -75,7 +75,7 @@ public FilterAggregatorFactory(
      * necessary. This is done lazily so that the {@link Weight} is only created
      * if the aggregation collects documents reducing the overhead of the
      * aggregation in the case where no documents are collected.
-     *
+     * 
                            
      * Note that as aggregations are initialsed and executed in a serial manner,
      * no concurrency considerations are necessary here.
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
index 35d968b789a21..a8e157a1cbb79 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
@@ -92,7 +92,7 @@ public FiltersAggregatorFactory(
      * necessary. This is done lazily so that the {@link Weight}s are only
      * created if the aggregation collects documents reducing the overhead of
      * the aggregation in the case where no documents are collected.
-     *
+     * 
                            
      * Note: With concurrent segment search use case, multiple aggregation collectors executing
      * on different threads will try to fetch the weights. To handle the race condition there is
      * a synchronization block
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
index d7f89225524c0..b4f1e78f77aaf 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
@@ -546,7 +546,7 @@ private int collectValue(long owningBucketOrd, int roundingIdx, int doc, long ro
 
                 /**
                  * Increase the rounding of {@code owningBucketOrd} using
-                 * estimated, bucket counts, {@link #rebucket() rebucketing} the all
+                 * estimated, bucket counts, {@link FromMany#rebucket()} rebucketing} the all
                  * buckets if the estimated number of wasted buckets is too high.
                  */
                 private int increaseRoundingIfNeeded(long owningBucketOrd, int oldEstimatedBucketCount, long newKey, int oldRounding) {
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index 181fb468f3356..a978b5cfa3e3e 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -245,7 +245,7 @@ public DateHistogramAggregationBuilder dateHistogramInterval(DateHistogramInterv
     /**
      * Sets the interval of the DateHistogram using calendar units (`1d`, `1w`, `1M`, etc).  These units
      * are calendar-aware, meaning they respect leap additions, variable days per month, etc.
-     *
+     * 
                            
      * This is mutually exclusive with {@link DateHistogramAggregationBuilder#fixedInterval(DateHistogramInterval)}
      *
      * @param interval The calendar interval to use with the aggregation
@@ -258,7 +258,7 @@ public DateHistogramAggregationBuilder calendarInterval(DateHistogramInterval in
     /**
      * Sets the interval of the DateHistogram using fixed units (`1ms`, `1s`, `10m`, `4h`, etc).  These are
      * not calendar aware and are simply multiples of fixed, SI units.
-     *
+     * 
                            
      * This is mutually exclusive with {@link DateHistogramAggregationBuilder#calendarInterval(DateHistogramInterval)}
      *
      * @param interval The fixed interval to use with the aggregation
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index 8233c3d995dda..f602eea7a9b12 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -216,9 +216,7 @@ public void collectDebugInfo(BiConsumer add) {
     }
 
     /**
-     * Returns the size of the bucket in specified units.
-     *
-     * If unitSize is null, returns 1.0
+     * @return the size of the bucket in specified units, or 1.0 if unitSize is null
      */
     @Override
     public double bucketSize(long bucket, Rounding.DateTimeUnit unitSize) {
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java
index 8f8e71f3cd685..9f907bcacadf9 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramInterval.java
@@ -129,7 +129,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
      * Converts this DateHistogramInterval into a millisecond representation.  If this is a calendar
      * interval, it is an approximation of milliseconds based on the fixed equivalent (e.g. `1h` is treated as 60
      * fixed minutes, rather than the hour at a specific point in time.
-     *
+     * 
                            
      * This is merely a convenience helper for quick comparisons and should not be used for situations that
      * require precise durations.
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalConsumer.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalConsumer.java
index a2c63cf25c0c2..1d21152b6f622 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalConsumer.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalConsumer.java
@@ -34,7 +34,7 @@
 
 /**
  * A shared interface for aggregations that parse and use "interval" parameters.
- *
+ * 
                            
  * Provides definitions for the new fixed and calendar intervals, and deprecated
  * defintions for the old interval/dateHisto interval parameters
  *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
index 2f8d512b7511f..83c20ba1c1d04 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateIntervalWrapper.java
@@ -53,13 +53,13 @@
 
 /**
  * A class that handles all the parsing, bwc and deprecations surrounding date histogram intervals.
- *
+ * 
                            
  * - Provides parser helpers for the deprecated interval/dateHistogramInterval parameters.
  * - Provides parser helpers for the new calendar/fixed interval parameters
  * - Can read old intervals from a stream and convert to new intervals
  * - Can write new intervals to old format when streaming out
  * - Provides a variety of helper methods to interpret the intervals as different types, depending on caller's need
- *
+ * 
                            
  * After the deprecated parameters are removed, this class can be simplified greatly.  The legacy options
  * will be removed, and the mutual-exclusion checks can be done in the setters directly removing the need
  * for the enum and the complicated "state machine" logic
@@ -220,7 +220,7 @@ public DateHistogramInterval getAsCalendarInterval() {
     /**
      * Sets the interval of the DateHistogram using calendar units (`1d`, `1w`, `1M`, etc).  These units
      * are calendar-aware, meaning they respect leap additions, variable days per month, etc.
-     *
+     * 
                            
      * This is mutually exclusive with {@link DateIntervalWrapper#fixedInterval(DateHistogramInterval)}
      *
      * @param interval The fixed interval to use
@@ -250,7 +250,7 @@ public DateHistogramInterval getAsFixedInterval() {
     /**
      * Sets the interval of the DateHistogram using fixed units (`1ms`, `1s`, `10m`, `4h`, etc).  These are
      * not calendar aware and are simply multiples of fixed, SI units.
-     *
+     * 
                            
      * This is mutually exclusive with {@link DateIntervalWrapper#calendarInterval(DateHistogramInterval)}
      *
      * @param interval The fixed interval to use
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java
index 69c70ed1bf7fd..235ae8fcee6d7 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DoubleBounds.java
@@ -49,7 +49,7 @@
 
 /**
  * Represent hard_bounds and extended_bounds in histogram aggregations.
- *
+ * 
                            
  * This class is similar to {@link LongBounds} used in date histograms, but is using longs to store data. LongBounds and DoubleBounds are
  * not used interchangeably and therefore don't share any common interfaces except for serialization.
  *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java
index 52f689eb7c229..8ebd67bc1ebe5 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java
@@ -491,7 +491,7 @@ private void mergeBucketsWithPlan(List buckets, List plan,
      * Makes a merge plan by simulating the merging of the two closest buckets, until the target number of buckets is reached.
      * Distance is determined by centroid comparison.
      * Then, this plan is actually executed and the underlying buckets are merged.
-     *
+     * 
                            
      * Requires: buckets is sorted by centroid.
      */
     private void mergeBucketsIfNeeded(List buckets, int targetNumBuckets, ReduceContext reduceContext) {
@@ -567,7 +567,7 @@ private void mergeBucketsWithSameMin(List buckets, ReduceContext reduceC
     /**
      * When two adjacent buckets A, B overlap (A.max > B.min) then their boundary is set to
      * the midpoint: (A.max + B.min) / 2.
-     *
+     * 
                            
      * After this adjustment, A will contain more values than indicated and B will have less.
      */
     private void adjustBoundsForOverlappingBuckets(List buckets, ReduceContext reduceContext) {
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java
index e138824e7fce8..ad6572916c84a 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/LongBounds.java
@@ -54,7 +54,7 @@
 
 /**
  * Represent hard_bounds and extended_bounds in date-histogram aggregations.
- *
+ * 
                            
  * This class is similar to {@link DoubleBounds} used in histograms, but is using longs to store data. LongBounds and DoubleBounds are
  *  * not used interchangeably and therefore don't share any common interfaces except for serialization.
  *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java
index 11ff7dbc407cb..526945243c786 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java
@@ -76,7 +76,7 @@ public class VariableWidthHistogramAggregator extends DeferableBucketAggregator
     /**
      * This aggregator goes through multiple phases of collection. Each phase has a different CollectionPhase::collectValue
      * implementation
-     *
+     * 
                            
      * Running a clustering algorithm like K-Means is unfeasible because large indices don't fit into memory.
      * But having multiple collection phases lets us accurately bucket the docs in one pass.
      */
@@ -231,7 +231,7 @@ protected void swap(int i, int j) {
              * Produces a merge map where `mergeMap[i]` represents the index that values[i]
              * would be moved to if values were sorted
              * In other words, this method produces a merge map that will sort values
-             *
+             * 
                            
              * See BucketsAggregator::mergeBuckets to learn more about the merge map
              */
             public long[] generateMergeMap() {
@@ -242,10 +242,10 @@ public long[] generateMergeMap() {
 
         /**
          * Sorting the documents by key lets us bucket the documents into groups with a single linear scan
-         *
+         * 
                            
          * But we can't do this by just sorting buffer, because we also need to generate a merge map
          * for every change we make to the list, so that we can apply the changes to the underlying buckets as well.
-         *
+         * 
                            
          * By just creating a merge map, we eliminate the need to actually sort buffer. We can just
          * use the merge map to find any doc's sorted index.
          */
@@ -347,7 +347,7 @@ private void createAndAppendNewCluster(double value) {
         /**
          * Move the last cluster to position idx
          * This is expensive because a merge map of size numClusters is created, so don't call this method too often
-         *
+         * 
                            
          * TODO: Make this more efficient
          */
         private void moveLastCluster(int index) {
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregator.java
index 23e2bc6b3e54b..a886bdb3ae188 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregator.java
@@ -55,7 +55,7 @@
 
 /**
  * Aggregate on only the top-scoring docs on a shard.
- *
+ * 
                            
  * TODO currently the diversity feature of this agg offers only 'script' and
  * 'field' as a means of generating a de-dup value. In future it would be nice
  * if users could use any of the "bucket" aggs syntax (geo, date histogram...)
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java
index 4d1cbd4ce72f1..5b90163fa3959 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalMultiTerms.java
@@ -407,8 +407,8 @@ public int hashCode() {
 
     /**
      * Copy from InternalComposite
-     *
-     * Format obj using the provided {@link DocValueFormat}.
+     * 
                            
+     * Format {@code obj} using the provided {@link DocValueFormat}.
      * If the format is equals to {@link DocValueFormat#RAW}, the object is returned as is
      * for numbers and a string for {@link BytesRef}s.
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java
index 341ac021f6e0c..0e773291881cf 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -436,11 +436,11 @@ public InternalAggregation reduce(List aggregations, Reduce
         }
 
         final List reducedBuckets;
-        /**
-         * Buckets returned by a partial reduce or a shard response are sorted by key.
-         * That allows to perform a merge sort when reducing multiple aggregations together.
-         * For backward compatibility, we disable the merge sort and use ({@link InternalTerms#reduceLegacy} if any of
-         * the provided aggregations use a different {@link InternalTerms#reduceOrder}.
+        /*
+          Buckets returned by a partial reduce or a shard response are sorted by key.
+          That allows to perform a merge sort when reducing multiple aggregations together.
+          For backward compatibility, we disable the merge sort and use ({@link InternalTerms#reduceLegacy} if any of
+          the provided aggregations use a different {@link InternalTerms#reduceOrder}.
          */
         BucketOrder thisReduceOrder = getReduceOrder(aggregations);
         if (isKeyOrder(thisReduceOrder)) {
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java
index 0482ef823818c..59f48bd7fbaba 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java
@@ -94,10 +94,10 @@ public MultiTermsAggregator(
         this.partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(b -> b.bucketOrd, this);
         // Todo, copy from TermsAggregator. need to remove duplicate code.
         if (subAggsNeedScore() && descendsFromNestedAggregator(parent)) {
-            /**
-             * Force the execution to depth_first because we need to access the score of
-             * nested documents in a sub-aggregation and we are not able to generate this score
-             * while replaying deferred documents.
+            /*
+              Force the execution to depth_first because we need to access the score of
+              nested documents in a sub-aggregation and we are not able to generate this score
+              while replaying deferred documents.
              */
             this.collectMode = SubAggCollectionMode.DEPTH_FIRST;
         } else {
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
index dc616ca7512be..5d83d926ab36f 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
@@ -196,7 +196,7 @@ public double getPrecision() {
      * Set's the false-positive rate for individual cuckoo filters.  Does not dictate the overall fpp rate
      * since we use a "scaling" cuckoo filter which adds more filters as required, and the overall
      * error rate grows differently than individual filters
-     *
+     * 
                            
      * This value does, however, affect the overall space usage of the filter.  Coarser precisions provide
      * more compact filters.  The default is 0.01
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java
index 7cacf1e918380..845149d894aad 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregator.java
@@ -244,10 +244,10 @@ public TermsAggregator(
         partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(b -> b.bucketOrd, this);
         this.format = format;
         if (subAggsNeedScore() && descendsFromNestedAggregator(parent)) {
-            /**
-             * Force the execution to depth_first because we need to access the score of
-             * nested documents in a sub-aggregation and we are not able to generate this score
-             * while replaying deferred documents.
+            /*
+              Force the execution to depth_first because we need to access the score of
+              nested documents in a sub-aggregation and we are not able to generate this score
+              while replaying deferred documents.
              */
             this.collectMode = SubAggCollectionMode.DEPTH_FIRST;
         } else {
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLog.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLog.java
index 6b998fc86361d..902e4d69ed5fa 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLog.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLog.java
@@ -34,9 +34,9 @@
 
 /**
  * Hyperloglog counter, implemented based on pseudo code from
- * http://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/40671.pdf and its appendix
- * https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen
- *
+ * 40671.pdf and its
+ * appendix
+ * 
                            
  * Trying to understand what this class does without having read the paper is considered adventurous.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java
index 4354a23b70f6b..e74179b403e8e 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java
@@ -46,7 +46,6 @@
 
 /**
  * Base class for HLL++ algorithms.
- *
  * It contains methods for cloning and serializing the data structure.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractLinearCounting.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractLinearCounting.java
index 3f5f524c9c2f5..7c00b25ae365f 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractLinearCounting.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractLinearCounting.java
@@ -36,11 +36,11 @@
 
 /**
  * Linear counter, implemented based on pseudo code from
- * http://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/40671.pdf and its appendix
- * https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen
- *
+ * 40671.pdf and its
+ * appendix
+ * 
                            
  * Trying to understand what this class does without having read the paper is considered adventurous.
- *
+ * 
                            
  * The algorithm just keep a record of all distinct values provided encoded as an integer.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java
index 762b10fc28fa7..fa8830de5dab9 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java
@@ -73,7 +73,7 @@ public static > ConstructingO
         ParseField valuesField
     ) {
 
-        /**
+        /*
          * This is a non-ideal ConstructingObjectParser, because it is a compromise between Percentiles and Ranks.
          * Ranks requires an array of values because there is no sane default, and we want to keep that in the ctor.
          * Percentiles has defaults, which means the API allows the user to either use the default or configure
@@ -86,6 +86,7 @@ public static > ConstructingO
          * out the behavior from there
          *
          * `args` are provided from the ConstructingObjectParser in-order they are defined in the parser.  So:
+         *
          *  - args[0]: values
          *  - args[1]: tdigest config options
          *  - args[2]: hdr config options
@@ -197,7 +198,7 @@ public boolean keyed() {
     /**
      * Expert: set the number of significant digits in the values. Only relevant
      * when using {@link PercentilesMethod#HDR}.
-     *
+     * 
                            
      * Deprecated: set numberOfSignificantValueDigits by configuring a {@link PercentilesConfig.Hdr} instead
      * and set via {@link PercentilesAggregationBuilder#percentilesConfig(PercentilesConfig)}
      */
@@ -217,7 +218,7 @@ public T numberOfSignificantValueDigits(int numberOfSignificantValueDigits) {
     /**
      * Expert: get the number of significant digits in the values. Only relevant
      * when using {@link PercentilesMethod#HDR}.
-     *
+     * 
                            
      * Deprecated: get numberOfSignificantValueDigits by inspecting the {@link PercentilesConfig} returned from
      * {@link PercentilesAggregationBuilder#percentilesConfig()} instead
      */
@@ -232,7 +233,7 @@ public int numberOfSignificantValueDigits() {
     /**
      * Expert: set the compression. Higher values improve accuracy but also
      * memory usage. Only relevant when using {@link PercentilesMethod#TDIGEST}.
-     *
+     * 
                            
      * Deprecated: set compression by configuring a {@link PercentilesConfig.TDigest} instead
      * and set via {@link PercentilesAggregationBuilder#percentilesConfig(PercentilesConfig)}
      */
@@ -249,7 +250,7 @@ public T compression(double compression) {
     /**
      * Expert: get the compression. Higher values improve accuracy but also
      * memory usage. Only relevant when using {@link PercentilesMethod#TDIGEST}.
-     *
+     * 
                            
      * Deprecated: get compression by inspecting the {@link PercentilesConfig} returned from
      * {@link PercentilesAggregationBuilder#percentilesConfig()} instead
      */
@@ -317,15 +318,15 @@ public T percentilesConfig(PercentilesConfig percentilesConfig) {
 
     /**
      * Return the current algo configuration, or a default (Tdigest) otherwise
-     *
+     * 
                            
      * This is needed because builders don't have a "build" or "finalize" method, but
      * the old API did bake in defaults.  Certain operations like xcontent, equals, hashcode
      * will use the values in the builder at any time and need to be aware of defaults.
-     *
+     * 
                            
      * But to maintain BWC behavior as much as possible, we allow the user to set
      * algo settings independent of method.  To keep life simple we use a null to track
      * if any method has been selected yet.
-     *
+     * 
                            
      * However, this means we need a way to fetch the default if the user hasn't
      * selected any method and uses a builder-side feature like xcontent
      */
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java
index 7bd1e1aa22a90..7ab35eaed785c 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlus.java
@@ -49,16 +49,16 @@
  * Hyperloglog++ counter, implemented based on pseudo code from
  * http://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/40671.pdf and its appendix
  * https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen
- *
+ * 
                            
  * This implementation is different from the original implementation in that it uses a hash table instead of a sorted list for linear
  * counting. Although this requires more space and makes hyperloglog (which is less accurate) used sooner, this is also considerably faster.
- *
+ * 
                            
  * Trying to understand what this class does without having read the paper is considered adventurous.
- *
+ * 
                            
  * The HyperLogLogPlusPlus contains two algorithms, one for linear counting and the HyperLogLog algorithm. Initially hashes added to the
  * data structure are processed using the linear counting until a threshold defined by the precision is reached where the data is replayed
  * to the HyperLogLog algorithm and then this is used.
- *
+ * 
                            
  * It supports storing several HyperLogLogPlusPlus structures which are identified by a bucket number.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java
index 252df7358ddcd..558e9df93c804 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparse.java
@@ -40,7 +40,7 @@
 /**
  * AbstractHyperLogLogPlusPlus instance that only supports linear counting. The maximum number of hashes supported
  * by the structure is determined at construction time.
- *
+ * 
                            
  * This structure expects all the added values to be distinct and therefore there are no checks
  * if an element has been previously added.
  *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java
index be98df384fc28..6f9be06231819 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java
@@ -52,7 +52,7 @@
 
 /**
  * A field data based aggregator that counts the number of values a specific field has within the aggregation context.
- *
+ * 
                            
  * This aggregator works in a multi-bucket mode, that is, when serves as a sub-aggregator, a single aggregator instance aggregates the
  * counts for all buckets owned by the parent aggregator)
  *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java
index 3015e7ce9f364..c7f2a29793bff 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/BucketHelpers.java
@@ -64,7 +64,7 @@ public class BucketHelpers {
      * a date_histogram might have empty buckets due to no data existing for that time interval.
      * This can cause problems for operations like a derivative, which relies on a continuous
      * function.
-     *
+     * 
                            
      * "insert_zeros": empty buckets will be filled with zeros for all metrics
      * "skip": empty buckets will simply be ignored
      *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregator.java
index a4c3c14f3365f..3e97eda693b91 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovFnPipelineAggregator.java
@@ -52,18 +52,18 @@
 /**
  * This pipeline aggregation gives the user the ability to script functions that "move" across a window
  * of data, instead of single data points.  It is the scripted version of MovingAvg pipeline agg.
- *
+ * 
                            
  * Through custom script contexts, we expose a number of convenience methods:
- *
- *  - max
- *  - min
- *  - sum
- *  - unweightedAvg
- *  - linearWeightedAvg
- *  - ewma
- *  - holt
- *  - holtWintersMovAvg
- *
+ * 
                              
+ *   
                            • max
                              • 
+ *  
                            •  min
                              • 
+ *  
                            •  sum
                              • 
+ *  
                            •  unweightedAvg
                              • 
+ *  
                            •  linearWeightedAvg
                              • 
+ *  
                            •  ewma
                              • 
+ *  
                            •  holt
                              • 
+ *  
                            •  holtWintersMovAvg
                              • 
+ * 
                            
  *  The user can also define any arbitrary logic via their own scripting, or combine with the above methods.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovingFunctions.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovingFunctions.java
index bac486576f537..051b9c43f63f5 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovingFunctions.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/MovingFunctions.java
@@ -75,7 +75,7 @@ public static double sum(double[] values) {
 
     /**
      * Calculate a simple unweighted (arithmetic) moving average.
-     *
+     * 
                            
      * Only finite values are averaged.  NaN or null are ignored.
      * If all values are missing/null/NaN, the return value will be NaN.
      * The average is based on the count of non-null, non-NaN values.
@@ -94,7 +94,7 @@ public static double unweightedAvg(double[] values) {
 
     /**
      * Calculate a standard deviation over the values using the provided average.
-     *
+     * 
                            
      * Only finite values are averaged.  NaN or null are ignored.
      * If all values are missing/null/NaN, the return value will be NaN.
      * The average is based on the count of non-null, non-NaN values.
@@ -118,7 +118,7 @@ public static double stdDev(double[] values, double avg) {
     /**
      * Calculate a linearly weighted moving average, such that older values are
      * linearly less important.  "Time" is determined by position in collection
-     *
+     * 
                            
      * Only finite values are averaged.  NaN or null are ignored.
      * If all values are missing/null/NaN, the return value will be NaN
      * The average is based on the count of non-null, non-NaN values.
@@ -141,11 +141,11 @@ public static double linearWeightedAvg(double[] values) {
     /**
      *
      * Calculate a exponentially weighted moving average.
-     *
+     * 
                            
      * Alpha controls the smoothing of the data.  Alpha = 1 retains no memory of past values
      * (e.g. a random walk), while alpha = 0 retains infinite memory of past values (e.g.
      * the series mean).  Useful values are somewhere in between.  Defaults to 0.5.
-     *
+     * 
                            
      * Only finite values are averaged.  NaN or null are ignored.
      * If all values are missing/null/NaN, the return value will be NaN
      * The average is based on the count of non-null, non-NaN values.
@@ -171,13 +171,13 @@ public static double ewma(double[] values, double alpha) {
 
     /**
      * Calculate a doubly exponential weighted moving average
-     *
+     * 
                            
      * Alpha controls the smoothing of the data.  Alpha = 1 retains no memory of past values
      * (e.g. a random walk), while alpha = 0 retains infinite memory of past values (e.g.
      * the series mean).  Useful values are somewhere in between.  Defaults to 0.5.
-     *
+     * 
                            
      * Beta is equivalent to alpha, but controls the smoothing of the trend instead of the data
-     *
+     * 
                            
      * Only finite values are averaged.  NaN or null are ignored.
      * If all values are missing/null/NaN, the return value will be NaN
      * The average is based on the count of non-null, non-NaN values.
@@ -241,14 +241,14 @@ public static double[] holtForecast(double[] values, double alpha, double beta,
 
     /**
      * Calculate a triple exponential weighted moving average
-     *
+     * 
                            
      * Alpha controls the smoothing of the data.  Alpha = 1 retains no memory of past values
      * (e.g. a random walk), while alpha = 0 retains infinite memory of past values (e.g.
      * the series mean).  Useful values are somewhere in between.  Defaults to 0.5.
-     *
+     * 
                            
      * Beta is equivalent to alpha, but controls the smoothing of the trend instead of the data.
      * Gamma is equivalent to alpha, but controls the smoothing of the seasonality instead of the data
-     *
+     * 
                            
      * Only finite values are averaged.  NaN or null are ignored.
      * If all values are missing/null/NaN, the return value will be NaN
      * The average is based on the count of non-null, non-NaN values.
diff --git a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimulatedAnealingMinimizer.java b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimulatedAnealingMinimizer.java
index a61a866228161..8427346357b0e 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimulatedAnealingMinimizer.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/pipeline/SimulatedAnealingMinimizer.java
@@ -36,11 +36,11 @@
 
 /**
  * A cost minimizer which will fit a MovAvgModel to the data.
- *
+ * 
                            
  * This optimizer uses naive simulated annealing.  Random solutions in the problem space
  * are generated, compared against the last period of data, and the least absolute deviation
  * is recorded as a cost.
- *
+ * 
                            
  * If the new cost is better than the old cost, the new coefficients are chosen.  If the new
  * solution is worse, there is a temperature-dependent probability it will be randomly selected
  * anyway.  This allows the algo to sample the problem space widely.  As iterations progress,
@@ -114,7 +114,7 @@ private static double acceptanceProbability(double oldCost, double newCost, doub
     /**
      * Calculates the "cost" of a model.  E.g. when run on the training data, how closely do the  predictions
      * match the test data
-     *
+     * 
                            
      * Uses Least Absolute Differences to calculate error.  Note that this is not scale free, but seems
      * to work fairly well in practice
      *
diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java
index 25cdd76183602..a7de47bed2e6e 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java
@@ -78,9 +78,9 @@
  * Provides a set of static helpers to determine if a particular type of InternalAggregation "has a value"
  * or not.  This can be difficult to determine from an external perspective because each agg uses
  * different internal bookkeeping to determine if it is empty or not (NaN, +/-Inf, 0.0, etc).
- *
+ * 
                            
  * This set of helpers aim to ease that task by codifying what "empty" is for each agg.
- *
+ * 
                            
  * It is not entirely accurate for all aggs, since some do not expose or track the needed state
  * (e.g. sum doesn't record count, so it's not clear if the sum is 0.0 because it is empty
  * or because of summing to zero).  Pipeline aggs in particular are not well supported
diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java
index 00c769c015fbc..f6d6fe28a56d3 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/support/CoreValuesSourceType.java
@@ -274,7 +274,7 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing,
          * MappedFieldType, it prefers to get the formatter from there. Only when a field can't be
          * resolved (which is to say script cases and unmapped field cases), it will fall back to calling this method on whatever
          * ValuesSourceType it was able to resolve to.
-         *
+         * 
                            
          * For geoshape field we may never hit this function till we have aggregations which are only geo_shape
          * specific and not present on geo_points, as we use default CoreValueSource types for Geo based aggregations
          * as GEOPOINT
diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
index e3f914ca259f6..c866238d12fcb 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -51,7 +51,7 @@
 /**
  * Similar to {@link ValuesSourceAggregationBuilder}, except it references multiple ValuesSources (e.g. so that an aggregation
  * can pull values from multiple fields).
- *
+ * 
                            
  * A limitation of this class is that all the ValuesSource's being refereenced must be of the same type.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 70382369d5615..7a73fafb4a809 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -264,7 +264,7 @@ protected final void doWriteTo(StreamOutput out) throws IOException {
 
     /**
      * DO NOT OVERRIDE THIS!
-     *
+     * 
                            
      * This method only exists for legacy support.  No new aggregations need this, nor should they override it.
      *
      * @param version For backwards compatibility, subclasses can change behavior based on the version
diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceType.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceType.java
index 86102e63297d1..9158e9f59cab2 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceType.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceType.java
@@ -43,14 +43,14 @@
  * {@link ValuesSourceType} represents a collection of fields that share a common set of operations, for example all numeric fields.
  * Aggregations declare their support for a given ValuesSourceType (via {@link ValuesSourceRegistry.Builder#register}),
  * and should then not need to care about the fields which use that ValuesSourceType.
- *
+ * 
                            
  * ValuesSourceTypes provide a set of methods to instantiate concrete {@link ValuesSource} instances, based on the actual source of the
  * data for the aggregations.  In general, aggregations should not call these methods, but rather rely on {@link ValuesSourceConfig} to have
  * selected the correct implementation.
- *
+ * 
                            
  * ValuesSourceTypes should be stateless.  We recommend that plugins define an enum for their ValuesSourceTypes, even if the plugin only
  * intends to define one ValuesSourceType.  ValuesSourceTypes are not serialized as part of the aggregations framework.
- *
+ * 
                            
  * Prefer reusing an existing ValuesSourceType (ideally from {@link CoreValuesSourceType}) over creating a new type.  There are some cases
  * where creating a new type is necessary however.  In particular, consider a new ValuesSourceType if the field has custom encoding/decoding
  * requirements; if the field needs to expose additional information to the aggregation (e.g. {@link ValuesSource.Range#rangeType()}); or
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index 29fbee416a9b8..ebf9623eb367a 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -255,8 +255,8 @@ boolean isNodeInDuress() {
         return isNodeInDuress;
     }
 
-    /**
-     * Returns true if the increase in heap usage is due to search requests.
+    /*
+      Returns true if the increase in heap usage is due to search requests.
      */
 
     /**
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
index d20e3e50d419f..4c28d96d8289e 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
@@ -54,7 +54,7 @@ private static class Defaults {
     /**
      * Defines the percentage of tasks to cancel relative to the number of successful task completions.
      * In other words, it is the number of tokens added to the bucket on each successful task completion.
-     *
+     * 
                            
      * The setting below is deprecated.
      * To keep backwards compatibility, the old usage is remained, and it's also used as the fallback for the new usage.
      */
@@ -71,7 +71,7 @@ private static class Defaults {
     /**
      * Defines the number of tasks to cancel per unit time (in millis).
      * In other words, it is the number of tokens added to the bucket each millisecond.
-     *
+     * 
                            
      * The setting below is deprecated.
      * To keep backwards compatibility, the old usage is remained, and it's also used as the fallback for the new usage.
      */
@@ -86,7 +86,7 @@ private static class Defaults {
 
     /**
      * Defines the maximum number of tasks that can be cancelled before being rate-limited.
-     *
+     * 
                            
      * The setting below is deprecated.
      * To keep backwards compatibility, the old usage is remained, and it's also used as the fallback for the new usage.
      */
diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchSubPhase.java
index fa30b2a5c7450..f3a1d5cafe755 100644
--- a/server/src/main/java/org/opensearch/search/fetch/FetchSubPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/FetchSubPhase.java
@@ -88,7 +88,7 @@ public int docId() {
         /**
          * This lookup provides access to the source for the given hit document. Note
          * that it should always be set to the correct doc ID and {@link LeafReaderContext}.
-         *
+         * 
                            
          * In most cases, the hit document's source is loaded eagerly at the start of the
          * {@link FetchPhase}. This lookup will contain the preloaded source.
          */
@@ -103,7 +103,7 @@ public IndexReader topLevelReader() {
 
     /**
      * Returns a {@link FetchSubPhaseProcessor} for this sub phase.
-     *
+     * 
                            
      * If nothing should be executed for the provided {@code FetchContext}, then the
      * implementation should return {@code null}
      */
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchDocValuesPhase.java
index a8ab8c0dcb8a8..9b17d9dbcd8de 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchDocValuesPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchDocValuesPhase.java
@@ -47,7 +47,7 @@
 
 /**
  * Fetch sub phase which pulls data from doc values.
- *
+ * 
                            
  * Specifying {@code "docvalue_fields": ["field1", "field2"]}
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
index f50524244b115..cc941bb240b91 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
@@ -138,7 +138,7 @@ public SearchContext parentSearchContext() {
 
         /**
          * The _id of the root document.
-         *
+         * 
                            
          * Since this ID is available on the context, inner hits can avoid re-loading the root _id.
          */
         public String getId() {
@@ -151,7 +151,7 @@ public void setId(String id) {
 
         /**
          * A source lookup for the root document.
-         *
+         * 
                            
          * This shared lookup allows inner hits to avoid re-loading the root _source.
          */
         public SourceLookup getRootLookup() {
diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
index aa86ed4e56801..a520086de8051 100644
--- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
@@ -285,7 +285,7 @@ protected void search(List leaves, Weight weight, Collector c
 
     /**
      * Lower-level search API.
-     *
+     * 
                            
      * {@link LeafCollector#collect(int)} is called for every matching document in
      * the provided ctx.
      */
diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java
index dce6da897a74b..4ec21c469d9d3 100644
--- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java
@@ -286,7 +286,7 @@ public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) {
 
     /**
      * Indicates if the current index should perform frequent low level search cancellation check.
-     *
+     * 
                            
      * Enabling low-level checks will make long running searches to react to the cancellation request faster. However,
      * since it will produce more cancellation checks it might slow the search performance down.
      */
diff --git a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java
index 812030acb9561..734f62d048a5a 100644
--- a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java
+++ b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java
@@ -180,7 +180,7 @@ public List extractRawValues(String path) {
 
     /**
      * For the provided path, return its value in the source.
-     *
+     * 
                            
      * Note that in contrast with {@link SourceLookup#extractRawValues}, array and object values
      * can be returned.
      *
diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
index 9e5f2fa1592a4..bc82814e88bbd 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
@@ -31,7 +31,7 @@
 
 /**
  * TODO: Copied verbatim from {@link org.opensearch.ingest.PipelineConfiguration}.
- *
+ * 
                            
  * See if we can refactor into a common class. I suspect not, just because this one will hold
  */
 public class PipelineConfiguration extends AbstractDiffable implements ToXContentObject {
diff --git a/server/src/main/java/org/opensearch/search/pipeline/Processor.java b/server/src/main/java/org/opensearch/search/pipeline/Processor.java
index fb33f46acada4..0120d68ceb5aa 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/Processor.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/Processor.java
@@ -15,7 +15,7 @@
  * Whether changes are made and what exactly is modified is up to the implementation.
  * 
                            
  * Processors may get called concurrently and thus need to be thread-safe.
- *
+ * 
                            
  * TODO: Refactor {@link org.opensearch.ingest.Processor} to extend this interface, and specialize to IngestProcessor.
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java b/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java
index 0206b9b6cf716..0b80cdbef6669 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/ProcessorInfo.java
@@ -18,7 +18,7 @@
 
 /**
  * Information about a search pipeline  processor
- *
+ * 
                            
  * TODO: This is copy/pasted from the ingest ProcessorInfo.
  * Can/should we share implementation or is this just boilerplate?
  *
diff --git a/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java b/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java
index d1f525477f25f..904b04b249b1b 100644
--- a/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java
+++ b/server/src/main/java/org/opensearch/search/profile/AbstractInternalProfileTree.java
@@ -70,7 +70,7 @@ public AbstractInternalProfileTree() {
      * Returns a {@link QueryProfileBreakdown} for a scoring query.  Scoring queries (e.g. those
      * that are past the rewrite phase and are now being wrapped by createWeight() ) follow
      * a recursive progression.  We can track the dependency tree by a simple stack
-     *
+     * 
                            
      * The only hiccup is that the first scoring query will be identical to the last rewritten
      * query, so we need to take special care to fix that
      *
@@ -109,7 +109,7 @@ public PB getProfileBreakdown(E query) {
 
     /**
      * Helper method to add a new node to the dependency tree.
-     *
+     * 
                            
      * Initializes a new list in the dependency tree, saves the query and
      * generates a new {@link AbstractProfileBreakdown} to track the timings
      * of this element.
diff --git a/server/src/main/java/org/opensearch/search/profile/query/InternalProfileCollector.java b/server/src/main/java/org/opensearch/search/profile/query/InternalProfileCollector.java
index 8b860c3a58cea..024d91a8e2ed2 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/InternalProfileCollector.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/InternalProfileCollector.java
@@ -44,11 +44,12 @@
 
 /**
  * This class wraps a Lucene Collector and times the execution of:
- * - setScorer()
- * - collect()
- * - doSetNextReader()
- * - needsScores()
- *
+ * 
                              
+ * 
                            •  setScorer()
                              • 
+ * 
                            •  collect()
                              • 
+ * 
                            •  doSetNextReader()
                              • 
+ * 
                            •  needsScores()
                              • 
+ * 
                            
  * InternalProfiler facilitates the linking of the Collector graph
  *
  * @opensearch.internal
@@ -117,7 +118,7 @@ public Collector getCollector() {
 
     /**
      * Creates a human-friendly representation of the Collector name.
-     *
+     * 
                            
      * InternalBucket Collectors use the aggregation name in their toString() method,
      * which makes the profiled output a bit nicer.
      *
diff --git a/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java b/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java
index b1024c2312c50..a80ce1c658081 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java
@@ -44,7 +44,7 @@
  * "online" as the weights are wrapped by ContextIndexSearcher.  This allows us
  * to know the relationship between nodes in tree without explicitly
  * walking the tree or pre-wrapping everything
- *
+ * 
                            
  * A Profiler is associated with every Search, not per Search-Request. E.g. a
  * request may execute two searches (query + global agg).  A Profiler just
  * represents one of those
diff --git a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
index 7de605a244d09..f8a1e99ff585f 100644
--- a/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/opensearch/search/query/QuerySearchResult.java
@@ -123,7 +123,7 @@ public static QuerySearchResult nullInstance() {
      * Returns true if the result doesn't contain any useful information.
      * It is used by the search action to avoid creating an empty response on
      * shard request that rewrites to match_no_docs.
-     *
+     * 
                            
      * TODO: Currently we need the concrete aggregators to build empty responses. This means that we cannot
      *       build an empty response in the coordinating node so we rely on this hack to ensure that at least one shard
      *       returns a valid empty response. We should move the ability to create empty responses to aggregation builders
diff --git a/server/src/main/java/org/opensearch/search/rescore/Rescorer.java b/server/src/main/java/org/opensearch/search/rescore/Rescorer.java
index 33f8e5e7b535d..ae025f70c95b3 100644
--- a/server/src/main/java/org/opensearch/search/rescore/Rescorer.java
+++ b/server/src/main/java/org/opensearch/search/rescore/Rescorer.java
@@ -41,7 +41,7 @@
 /**
  * A query rescorer interface used to re-rank the Top-K results of a previously
  * executed search.
- *
+ * 
                            
  * Subclasses should borrow heavily from {@link QueryRescorer} because it is
  * fairly well behaved and documents that tradeoffs that it is making. There
  * is also an {@code ExampleRescorer} that is worth looking at.
diff --git a/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java b/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java
index d7d554c058c37..856e103193463 100644
--- a/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java
+++ b/server/src/main/java/org/opensearch/search/slice/DocValuesSliceQuery.java
@@ -49,7 +49,7 @@
 
 /**
  * A {@link SliceQuery} that uses the numeric doc values of a field to do the slicing.
- *
+ * 
                            
  * NOTE: With deterministic field values this query can be used across different readers safely.
  * If updates are accepted on the field you must ensure that the same reader is used for all `slice` queries.
  *
diff --git a/server/src/main/java/org/opensearch/search/slice/TermsSliceQuery.java b/server/src/main/java/org/opensearch/search/slice/TermsSliceQuery.java
index 297020fe2fe4d..05f36b0d6f3cf 100644
--- a/server/src/main/java/org/opensearch/search/slice/TermsSliceQuery.java
+++ b/server/src/main/java/org/opensearch/search/slice/TermsSliceQuery.java
@@ -53,7 +53,7 @@
 
 /**
  * A {@link SliceQuery} that uses the terms dictionary of a field to do the slicing.
- *
+ * 
                            
  * NOTE: The cost of this filter is O(N*M) where N is the number of unique terms in the dictionary
  * and M is the average number of documents per term.
  * For each segment this filter enumerates the terms dictionary, computes the hash code for each term and fills
diff --git a/server/src/main/java/org/opensearch/search/suggest/Suggest.java b/server/src/main/java/org/opensearch/search/suggest/Suggest.java
index 17e5b44ec89e7..abd73b45ecc13 100644
--- a/server/src/main/java/org/opensearch/search/suggest/Suggest.java
+++ b/server/src/main/java/org/opensearch/search/suggest/Suggest.java
@@ -263,7 +263,7 @@ public void addTerm(T entry) {
         /**
          * Returns a integer representing the type of the suggestion. This is used for
          * internal serialization over the network.
-         *
+         * 
                            
          * This class is now serialized as a NamedWriteable and this method only remains for backwards compatibility
          */
         @Deprecated
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java
index 96e47cf7c8000..e3e6cad65be62 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestion.java
@@ -63,14 +63,14 @@
 
 /**
  * Suggestion response for {@link CompletionSuggester} results
- *
+ * 
                            
  * Response format for each entry:
  * {
  *     "text" : STRING
  *     "score" : FLOAT
  *     "contexts" : CONTEXTS
  * }
- *
+ * 
                            
  * CONTEXTS : {
  *     "CONTEXT_NAME" : ARRAY,
  *     ..
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java b/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java
index 56cc8fbfbcf66..139742f84b80b 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/FuzzyOptions.java
@@ -142,7 +142,7 @@ public int getEditDistance() {
 
     /**
      * Returns if transpositions option is set
-     *
+     * 
                            
      * if transpositions is set, then swapping one character for another counts as one edit instead of two.
      */
     public boolean isTranspositions() {
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/TopSuggestGroupDocsCollector.java b/server/src/main/java/org/opensearch/search/suggest/completion/TopSuggestGroupDocsCollector.java
index 4fbd661037aa9..fbc39536502de 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/TopSuggestGroupDocsCollector.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/TopSuggestGroupDocsCollector.java
@@ -43,7 +43,7 @@
 /**
  *
  * Extension of the {@link TopSuggestDocsCollector} that returns top documents from the completion suggester.
- *
+ * 
                            
  * This collector groups suggestions coming from the same document but matching different contexts
  * or surface form together. When different contexts or surface forms match the same suggestion form only
  * the best one per document (sorted by weight) is kept.
@@ -55,7 +55,7 @@ class TopSuggestGroupDocsCollector extends TopSuggestDocsCollector {
 
     /**
      * Sole constructor
-     *
+     * 
                            
      * Collects at most num completions
      * with corresponding document and weight
      */
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMapping.java b/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMapping.java
index 61e60293f4943..94707ff2b4569 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMapping.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMapping.java
@@ -54,7 +54,7 @@
 /**
  * A {@link ContextMapping} defines criteria that can be used to
  * filter and/or boost suggestions at query time for {@link CompletionFieldMapper}.
- *
+ * 
                            
  * Implementations have to define how contexts are parsed at query/index time
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMappings.java b/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMappings.java
index 0f5781fefcf07..cff5a901a473f 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMappings.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/context/ContextMappings.java
@@ -124,11 +124,11 @@ public Iterator> iterator() {
      * Field prepends context values with a suggestion
      * Context values are associated with a type, denoted by
      * a type id, which is prepended to the context value.
-     *
+     * 
                            
      * Every defined context mapping yields a unique type id (index of the
      * corresponding context mapping in the context mappings list)
      * for all its context values
-     *
+     * 
                            
      * The type, context and suggestion values are encoded as follows:
      * 
                            
      *     TYPE_ID | CONTEXT_VALUE | CONTEXT_SEP | SUGGESTION_VALUE
@@ -209,7 +209,7 @@ public ContextQuery toContextQuery(CompletionQuery query, Map
      * see {@link org.opensearch.search.suggest.completion.context.ContextMappings.TypedContextField}
      * @return a map of context names and their values
      *
@@ -232,7 +232,7 @@ public Map> getNamedContexts(List contexts) {
 
     /**
      * Loads {@link ContextMappings} from configuration
-     *
+     * 
                            
      * Expected configuration:
      *  List of maps representing {@link ContextMapping}
      *  [{"name": .., "type": .., ..}, {..}]
@@ -286,7 +286,7 @@ private static String extractRequiredValue(Map contextConfig, St
 
     /**
      * Writes a list of objects specified by the defined {@link ContextMapping}s
-     *
+     * 
                            
      * see {@link ContextMapping#toXContent(XContentBuilder, Params)}
      */
     @Override
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/context/GeoContextMapping.java b/server/src/main/java/org/opensearch/search/suggest/completion/context/GeoContextMapping.java
index 17f858b0b1302..0e29e928df760 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/context/GeoContextMapping.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/context/GeoContextMapping.java
@@ -71,7 +71,7 @@
  * The suggestions can be boosted and/or filtered depending on
  * whether it falls within an area, represented by a query geo hash
  * with a specified precision
- *
+ * 
                            
  * {@link GeoQueryContext} defines the options for constructing
  * a unit of query context for this context type
  *
diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java b/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java
index 6335629f61cf1..1a00cb9465771 100644
--- a/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java
+++ b/server/src/main/java/org/opensearch/search/suggest/phrase/DirectCandidateGenerator.java
@@ -147,9 +147,9 @@ public TermStats internalTermStats(BytesRef term) throws IOException {
         if (termsEnum.seekExact(term)) {
             return new TermStats(
                 termsEnum.docFreq(),
-                /**
-                 * We use the {@link TermsEnum#docFreq()} for fields that don't
-                 * record the {@link TermsEnum#totalTermFreq()}.
+                /*
+                  We use the {@link TermsEnum#docFreq()} for fields that don't
+                  record the {@link TermsEnum#totalTermFreq()}.
                  */
                 termsEnum.totalTermFreq() == -1 ? termsEnum.docFreq() : termsEnum.totalTermFreq()
             );
@@ -168,10 +168,10 @@ public CandidateSet drawCandidates(CandidateSet set) throws IOException {
         float origThreshold = spellchecker.getThresholdFrequency();
         try {
             if (suggestMode != SuggestMode.SUGGEST_ALWAYS) {
-                /**
-                 * We use the {@link TermStats#docFreq} to compute the frequency threshold
-                 * because that's what {@link DirectSpellChecker#suggestSimilar} expects
-                 * when filtering terms.
+                /*
+                  We use the {@link TermStats#docFreq} to compute the frequency threshold
+                  because that's what {@link DirectSpellChecker#suggestSimilar} expects
+                  when filtering terms.
                  */
                 int threshold = thresholdTermFrequency(original.termStats.docFreq);
                 if (threshold == Integer.MAX_VALUE) {
diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java b/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java
index f1dba9793ba9e..bc942da738c7e 100644
--- a/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java
+++ b/server/src/main/java/org/opensearch/search/suggest/phrase/LinearInterpolation.java
@@ -70,7 +70,7 @@ public final class LinearInterpolation extends SmoothingModel {
 
     /**
      * Creates a linear interpolation smoothing model.
-     *
+     * 
                            
      * Note: the lambdas must sum up to one.
      *
      * @param trigramLambda
diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index e37d964cc0424..a6bfb880cf249 100644
--- a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -222,7 +222,7 @@ public Integer gramSize() {
      * misspellings in order to form a correction. This method accepts a float
      * value in the range [0..1) as a fraction of the actual query terms a
      * number {@code >=1} as an absolute number of query terms.
-     *
+     * 
                            
      * The default is set to {@code 1.0} which corresponds to that only
      * corrections with at most 1 misspelled term are returned.
      */
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
index 1f063bfed8cfe..71918bc73b55a 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
@@ -1565,7 +1565,7 @@ private void runNextQueuedOperation(RepositoryData repositoryData, String reposi
 
     /**
      * Runs a cluster state update that checks whether we have outstanding snapshot deletions that can be executed and executes them.
-     *
+     * 
                            
      * TODO: optimize this to execute in a single CS update together with finalizing the latest snapshot
      */
     private void runReadyDeletions(RepositoryData repositoryData, String repository) {
@@ -2758,7 +2758,7 @@ public boolean assertAllListenersResolved() {
      * Every shard snapshot or clone state update can result in multiple snapshots being updated. In order to determine whether or not a
      * shard update has an effect we use an outer loop over all current executing snapshot operations that iterates over them in the order
      * they were started in and an inner loop over the list of shard update tasks.
-     *
+     * 
                            
      * If the inner loop finds that a shard update task applies to a given snapshot and either a shard-snapshot or shard-clone operation in
      * it then it will update the state of the snapshot entry accordingly. If that update was a noop, then the task is removed from the
      * iteration as it was already applied before and likely just arrived on the cluster-manager node again due to retries upstream.
@@ -2768,7 +2768,7 @@ public boolean assertAllListenersResolved() {
      * a task in the executed tasks collection applied to a shard it was waiting for to become available, then the shard snapshot operation
      * will be started for that snapshot entry and the task removed from the collection of tasks that need to be applied to snapshot
      * entries since it can not have any further effects.
-     *
+     * 
                            
      * Package private to allow for tests.
      */
     static final ClusterStateTaskExecutor SHARD_STATE_EXECUTOR = new ClusterStateTaskExecutor() {
@@ -3058,7 +3058,7 @@ private static ShardSnapshotStatus startShardSnapshotAfterClone(ClusterState cur
 
     /**
      * An update to the snapshot state of a shard.
-     *
+     * 
                            
      * Package private for testing
      */
     static final class ShardSnapshotUpdate {
diff --git a/server/src/main/java/org/opensearch/tasks/Task.java b/server/src/main/java/org/opensearch/tasks/Task.java
index dcd7fc581a8fd..ee2eae60eae8f 100644
--- a/server/src/main/java/org/opensearch/tasks/Task.java
+++ b/server/src/main/java/org/opensearch/tasks/Task.java
@@ -550,11 +550,11 @@ public int incrementResourceTrackingThreads() {
      * This method is called when threads finish execution, and also when the task is unregistered (to mark the task's
      * own thread as complete). When the active thread count becomes zero, the onTaskResourceTrackingCompleted method
      * is called exactly once on all registered listeners.
-     *
+     * 
                            
      * Since a task is unregistered after the message is processed, it implies that the threads responsible to produce
      * the response must have started prior to it (i.e. startThreadResourceTracking called before unregister).
      * This ensures that the number of active threads doesn't drop to zero pre-maturely.
-     *
+     * 
                            
      * Rarely, some threads may even start execution after the task is unregistered. As resource stats are piggy-backed
      * with the response, any thread usage info captured after the task is unregistered may be irrelevant.
      *
diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellation.java b/server/src/main/java/org/opensearch/tasks/TaskCancellation.java
index 9b61b2454b53c..2d152e513f197 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskCancellation.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskCancellation.java
@@ -18,7 +18,7 @@
 /**
  * TaskCancellation represents a task eligible for cancellation.
  * It doesn't guarantee that the task will actually get cancelled or not; that decision is left to the caller.
- *
+ * 
                            
  * It contains a list of cancellation reasons along with callbacks that are invoked when cancel() is called.
  *
  * @opensearch.internal
@@ -87,7 +87,7 @@ private void runOnCancelCallbacks() {
 
     /**
      * Returns the sum of all cancellation scores.
-     *
+     * 
                            
      * A zero score indicates no reason to cancel the task.
      * A task with a higher score suggests greater possibility of recovering the node when it is cancelled.
      */
diff --git a/server/src/main/java/org/opensearch/threadpool/Scheduler.java b/server/src/main/java/org/opensearch/threadpool/Scheduler.java
index 86c322ec89dd7..4a7c63a1b559a 100644
--- a/server/src/main/java/org/opensearch/threadpool/Scheduler.java
+++ b/server/src/main/java/org/opensearch/threadpool/Scheduler.java
@@ -60,7 +60,7 @@ public interface Scheduler {
 
     /**
      * Create a scheduler that can be used client side. Server side, please use ThreadPool.schedule instead.
-     *
+     * 
                            
      * Notice that if any scheduled jobs fail with an exception, these will bubble up to the uncaught exception handler where they will
      * be logged as a warning. This includes jobs started using execute, submit and schedule.
      * @param settings the settings to use
@@ -178,7 +178,7 @@ interface ScheduledCancellable extends Delayed, Cancellable {}
      * This class encapsulates the scheduling of a {@link Runnable} that needs to be repeated on a interval. For example, checking a value
      * for cleanup every second could be done by passing in a Runnable that can perform the check and the specified interval between
      * executions of this runnable. NOTE: the runnable is only rescheduled to run again after completion of the runnable.
-     *
+     * 
                            
      * For this class, completion means that the call to {@link Runnable#run()} returned or an exception was thrown and caught. In
      * case of an exception, this class will log the exception and reschedule the runnable for its next execution. This differs from the
      * {@link ScheduledThreadPoolExecutor#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)} semantics as an exception there would
diff --git a/server/src/main/java/org/opensearch/threadpool/TaskAwareRunnable.java b/server/src/main/java/org/opensearch/threadpool/TaskAwareRunnable.java
index 183b9b2f4cf9a..90f50f78d84ad 100644
--- a/server/src/main/java/org/opensearch/threadpool/TaskAwareRunnable.java
+++ b/server/src/main/java/org/opensearch/threadpool/TaskAwareRunnable.java
@@ -25,7 +25,7 @@
 /**
  * Responsible for wrapping the original task's runnable and sending updates on when it starts and finishes to
  * entities listening to the events.
- *
+ * 
                            
  * It's able to associate runnable with a task with the help of task Id available in thread context.
  */
 public class TaskAwareRunnable extends AbstractRunnable implements WrappedRunnable {
diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
index ecb5b2cef58ac..fab7620292dd2 100644
--- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
@@ -320,7 +320,7 @@ public ThreadPool(
 
     /**
      * Returns a value of milliseconds that may be used for relative time calculations.
-     *
+     * 
                            
      * This method should only be used for calculating time deltas. For an epoch based
      * timestamp, see {@link #absoluteTimeInMillis()}.
      */
@@ -330,7 +330,7 @@ public long relativeTimeInMillis() {
 
     /**
      * Returns a value of nanoseconds that may be used for relative time calculations.
-     *
+     * 
                            
      * This method should only be used for calculating time deltas. For an epoch based
      * timestamp, see {@link #absoluteTimeInMillis()}.
      */
@@ -343,7 +343,7 @@ public long relativeTimeInNanos() {
      * that require the highest precision possible. Performance critical code must use
      * either {@link #relativeTimeInNanos()} or {@link #relativeTimeInMillis()} which
      * give better performance at the cost of lower precision.
-     *
+     * 
                            
      * This method should only be used for calculating time deltas. For an epoch based
      * timestamp, see {@link #absoluteTimeInMillis()}.
      */
@@ -353,7 +353,7 @@ public long preciseRelativeTimeInNanos() {
 
     /**
      * Returns the value of milliseconds since UNIX epoch.
-     *
+     * 
                            
      * This method should only be used for exact date/time formatting. For calculating
      * time deltas that should not suffer from negative deltas, which are possible with
      * this method, see {@link #relativeTimeInMillis()}.
@@ -649,7 +649,7 @@ public String toString() {
     /**
      * A thread to cache millisecond time values from
      * {@link System#nanoTime()} and {@link System#currentTimeMillis()}.
-     *
+     * 
                            
      * The values are updated at a specified interval.
      */
     static class CachedTimeThread extends Thread {
diff --git a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java b/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java
index 5cb169439a14d..57707d3b44477 100644
--- a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java
+++ b/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java
@@ -48,11 +48,11 @@
  * requires that the underlying {@link DeflaterOutputStream} be closed to write EOS bytes. However, the
  * {@link BytesStream} should not be closed yet, as we have not used the bytes. This class handles these
  * intricacies.
- *
+ * 
                            
  * {@link CompressibleBytesOutputStream#materializeBytes()} should be called when all the bytes have been
  * written to this stream. If compression is enabled, the proper EOS bytes will be written at that point.
  * The underlying {@link BytesReference} will be returned.
- *
+ * 
                            
  * {@link CompressibleBytesOutputStream#close()} will NOT close the underlying stream. The byte stream passed
  * in the constructor must be closed individually.
  *
diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java
index 1599c4cb75517..8a5f6dfffb036 100644
--- a/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java
+++ b/server/src/main/java/org/opensearch/transport/RemoteClusterConnection.java
@@ -54,10 +54,10 @@
  * current node is part of the cluster and it won't receive cluster state updates from the remote cluster. Remote clusters are also not
  * fully connected with the current node. From a connection perspective a local cluster forms a bi-directional star network while in the
  * remote case we only connect to a subset of the nodes in the cluster in an uni-directional fashion.
- *
+ * 
                            
  * This class also handles the discovery of nodes from the remote cluster. The initial list of seed nodes is only used to discover all nodes
  * in the remote cluster and connects to all eligible nodes, for details see {@link RemoteClusterService#REMOTE_NODE_ATTRIBUTE}.
- *
+ * 
                            
  * In the case of a disconnection, this class will issue a re-connect task to establish at most
  * {@link SniffConnectionStrategy#REMOTE_CONNECTIONS_PER_CLUSTER} until either all eligible nodes are exhausted or the maximum number of
  * connections per cluster has been reached.
@@ -123,7 +123,7 @@ void ensureConnected(ActionListener listener) {
     /**
      * Collects all nodes on the connected cluster and returns / passes a nodeID to {@link DiscoveryNode} lookup function
      * that returns null if the node ID is not found.
-     *
+     * 
                            
      * The requests to get cluster state on the connected cluster are made in the system context because logically
      * they are equivalent to checking a single detail in the local cluster state and should not require that the
      * user who made the request that is using this method in its implementation is authorized to view the entire
diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java
index 32bedb52a9cef..de88c3619abe8 100644
--- a/server/src/main/java/org/opensearch/transport/TransportService.java
+++ b/server/src/main/java/org/opensearch/transport/TransportService.java
@@ -170,12 +170,12 @@ public void close() {}
     };
 
     static {
-        /**
-         * Registers server specific types as a streamables for serialization
-         * over the {@link StreamOutput} and {@link StreamInput} wire
+        /*
+          Registers server specific types as a streamables for serialization
+          over the {@link StreamOutput} and {@link StreamInput} wire
          */
         Streamables.registerStreamables();
-        /** Registers OpenSearch server specific exceptions (exceptions outside of core library) */
+        /* Registers OpenSearch server specific exceptions (exceptions outside of core library) */
         OpenSearchServerException.registerExceptions();
     }
 
diff --git a/server/src/main/java/org/opensearch/watcher/FileWatcher.java b/server/src/main/java/org/opensearch/watcher/FileWatcher.java
index 82f95d6a1622c..d773e3b5d7c9e 100644
--- a/server/src/main/java/org/opensearch/watcher/FileWatcher.java
+++ b/server/src/main/java/org/opensearch/watcher/FileWatcher.java
@@ -44,7 +44,7 @@
 
 /**
  * File resources watcher
- *
+ * 
                            
  * The file watcher checks directory and all its subdirectories for file changes and notifies its listeners accordingly
  *
  * @opensearch.internal
diff --git a/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java b/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java
index 9b9c00cd4252f..62002f9c6c323 100644
--- a/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java
+++ b/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java
@@ -49,7 +49,7 @@
 
 /**
  * Generic resource watcher service
- *
+ * 
                            
  * Other opensearch services can register their resource watchers with this service using {@link #add(ResourceWatcher)}
  * method. This service will call {@link org.opensearch.watcher.ResourceWatcher#checkAndNotify()} method of all
  * registered watcher periodically. The frequency of checks can be specified using {@code resource.reload.interval} setting, which
diff --git a/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java b/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java
index 7546a2d7c898e..d83539a2fef61 100644
--- a/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java
+++ b/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java
@@ -976,7 +976,7 @@ public void testFailureToAndFromXContentWithDetails() throws IOException {
 
     /**
      * Builds a {@link ToXContent} using a JSON XContentBuilder and compares the result to the given json in string format.
-     *
+     * 
                            
      * By default, the stack trace of the exception is not rendered. The parameter `errorTrace` forces the stack trace to
      * be rendered like the REST API does when the "error_trace" parameter is set to true.
      */
diff --git a/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java b/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java
index 2643aa5b6db01..6cbe458a35ef8 100644
--- a/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java
+++ b/server/src/test/java/org/opensearch/action/search/CreatePitControllerTests.java
@@ -339,8 +339,8 @@ public void onFailure(Exception e) {
                 createListener.onFailure(new Exception("Exception occurred in phase 1"));
                 latch.await();
                 assertEquals(0, updateNodesInvoked.size());
-                /**
-                 * cleanup is not called on create pit phase one failure
+                /*
+                  cleanup is not called on create pit phase one failure
                  */
                 assertEquals(0, deleteNodesInvoked.size());
             }
@@ -438,8 +438,8 @@ public void onFailure(Exception e) {
                 createListener.onResponse(searchResponse);
                 latch.await();
                 assertEquals(3, updateNodesInvoked.size());
-                /**
-                 * check if cleanup is called for all nodes in case of update pit failure
+                /*
+                  check if cleanup is called for all nodes in case of update pit failure
                  */
                 assertEquals(3, deleteNodesInvoked.size());
             }
@@ -526,8 +526,8 @@ public void onFailure(Exception e) {
                 createListener.onResponse(searchResponse);
                 latch.await();
                 assertEquals(3, updateNodesInvoked.size());
-                /**
-                 * check if cleanup is called for all nodes in case of update pit failure
+                /*
+                  check if cleanup is called for all nodes in case of update pit failure
                  */
                 assertEquals(3, deleteNodesInvoked.size());
             }
diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
index 03150d6e30755..dad0fa0efd3ec 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
@@ -153,7 +153,7 @@ public class TransportReplicationActionTests extends OpenSearchTestCase {
     /**
      * takes a request that was sent by a {@link TransportReplicationAction} and captured
      * and returns the underlying request if it's wrapped or the original (cast to the expected type).
-     *
+     * 
                            
      * This will throw a {@link ClassCastException} if the request is of the wrong type.
      */
     public static  R resolveRequest(TransportRequest requestOrWrappedRequest) {
diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
index 839ac7ab5bb92..cce8758ef1014 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
@@ -114,7 +114,7 @@
  * This test tests the concurrent execution of several transport replication actions. All of these actions (except one) acquire a single
  * permit during their execution on shards and are expected to fail if a global level or index level block is present in the cluster state.
  * These actions are all started at the same time, but some are delayed until one last action.
- *
+ * 
                            
  * This last action is special because it acquires all the permits on shards, adds the block to the cluster state and then "releases" the
  * previously delayed single permit actions. This way, there is a clear transition between the single permit actions executed before the
  * all permit action that sets the block and those executed afterwards that are doomed to fail because of the block.
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationPriorityTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationPriorityTests.java
index f9f181402da1b..8cd664c8c13fc 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationPriorityTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationPriorityTests.java
@@ -50,7 +50,7 @@ public class AllocationPriorityTests extends OpenSearchAllocationTestCase {
 
     /**
      * Tests that higher prioritized primaries and replicas are allocated first even on the balanced shard allocator
-     * See https://github.com/elastic/elasticsearch/issues/13249 for details
+     * See elasticsearch issue #13249 for details
      */
     public void testPrioritizedIndicesAllocatedFirst() {
         AllocationService allocation = createAllocationService(
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java
index ba1e2d27a3bc5..62dce9c4edeb5 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -247,13 +247,13 @@ public void testPrimaryBalanceWithPreferPrimaryBalanceSetting() {
     /**
      * This test verifies the allocation logic when nodes breach multiple constraints and ensure node breaching min
      * constraints chosen for allocation.
-     *
+     * 
                            
      * This test mimics a cluster state containing four nodes, where one node breaches two constraints while one breaches
      * only one. In order to have nodes breach constraints, test excludes two nodes (node2, node3) from allocation so
      * that other two nodes (node0, node1) have all shards assignments resulting in constraints breach. Test asserts that
      * the new primary shard assignment lands on the node breaching one constraint(node1), while replica land on the other
      * (node0). Final shard allocation state.
-     *
+     * 
                            
      routing_nodes:
      -----node_id[node2][V]
      -----node_id[node3][V]
@@ -384,13 +384,13 @@ public void testGlobalPrimaryBalance() throws Exception {
      * This test mimics a cluster state which can not be rebalanced due to
      * {@link org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider}
      * allocation decider which prevents shard relocation, leaving cluster unbalanced on primaries.
-     *
+     * 
                            
      * There are two nodes (N1, N2) where all primaries land on N1 while replicas on N2.
      * N1        N2
      * ------  --------
      * P1        R1
      * P2        R2
-     *
+     * 
                            
      * -----node_id[node_0][V]
      * --------[test][1], node[node_0], [P], s[STARTED], a[id=xqfZSToVSQaff2xvuxh_yA]
      * --------[test][0], node[node_0], [P], s[STARTED], a[id=VGjOeBGdSmu3pJR6T7v29A]
@@ -454,14 +454,14 @@ public void testPrimaryBalance_NotSolved_1() {
      * This test mimics cluster state where re-balancing is not possible due to existing limitation of re-balancing
      * logic which applies at index level i.e. balance shards single index across all nodes. This will be solved when
      * primary shard count across indices, constraint is added.
-     *
+     * 
                            
      * Please note, P1, P2 belongs to different index
-     *
+     * 
                            
      * N1        N2
      * ------  --------
      * P1       R1
      * P2       R2
-     *
+     * 
                            
      * -----node_id[node_0][V]
      * --------[test1][0], node[node_0], [P], s[STARTED], a[id=u7qtyy5AR42hgEa-JpeArg]
      * --------[test0][0], node[node_0], [P], s[STARTED], a[id=BQrLSo6sQyGlcLdVvGgqLQ]
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/CatAllocationTestCase.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/CatAllocationTestCase.java
index 10271cad33fec..8f90882c21804 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/CatAllocationTestCase.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/CatAllocationTestCase.java
@@ -66,7 +66,7 @@
  * A base testcase that allows to run tests based on the output of the CAT API
  * The input is a line based cat/shards output like:
  *   kibana-int           0 p STARTED       2  24.8kb 10.202.245.2 r5-9-35
- *
+ * 
                            
  * the test builds up a clusterstate from the cat input and optionally runs a full balance on it.
  * This can be used to debug cluster allocation decisions.
  */
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java
index cf32d2b3cf00f..7f2f048485318 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardConstraintDeciderOverlapTests.java
@@ -30,7 +30,7 @@ public class IndexShardConstraintDeciderOverlapTests extends OpenSearchAllocatio
     /**
      * High watermark breach blocks new shard allocations to affected nodes. If shard count on such
      * nodes is low, this will cause IndexShardPerNodeConstraint to breach.
-     *
+     * 
                            
      * This test verifies that this doesn't lead to unassigned shards, and there are no hot spots in eligible
      * nodes.
      */
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardHotSpotTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardHotSpotTests.java
index 2efbb256e36bc..617c9b4701722 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardHotSpotTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/IndexShardHotSpotTests.java
@@ -58,7 +58,7 @@ public void testUnderReplicatedClusterScaleOut() {
     /**
      * Test cluster scale in scenario, when nodes are gracefully excluded from
      * cluster before termination.
-     *
+     * 
                            
      * During moveShards(), shards are picked from across indexes in an interleaved manner.
      * This prevents hot spots by evenly picking up shards. Since shard order can change
      * in subsequent runs.
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java
index ef9ae90e18bb5..e1c0a7eff1f6e 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteShardsRebalanceShardsTests.java
@@ -21,7 +21,7 @@ public class RemoteShardsRebalanceShardsTests extends RemoteShardsBalancerBaseTe
 
     /**
      * Test remote shard allocation and balancing for standard new cluster setup.
-     *
+     * 
                            
      * Post rebalance primaries should be balanced across all the nodes.
      */
     public void testShardAllocationAndRebalance() {
@@ -72,7 +72,7 @@ private int getTotalShardCountAcrossNodes(final Map nodePrimari
 
     /**
      * Asserts that the expected value is within the variance range.
-     *
+     * 
                            
      * Being used to assert the average number of shards per node.
      * Variance is required in case of non-absolute mean values;
      * for example, total number of remote capable nodes in a cluster.
diff --git a/server/src/test/java/org/opensearch/common/RoundingTests.java b/server/src/test/java/org/opensearch/common/RoundingTests.java
index 0ebfe02dc7641..1a499bac3e2e8 100644
--- a/server/src/test/java/org/opensearch/common/RoundingTests.java
+++ b/server/src/test/java/org/opensearch/common/RoundingTests.java
@@ -239,7 +239,7 @@ public void testOffsetRounding() {
      * {@link DateTimeUnit} and {@link ZoneId} and often (50% of the time)
      * chooses test dates that are exactly on or close to offset changes (e.g.
      * DST) in the chosen time zone.
-     *
+     * 
                            
      * It rounds the test date down and up and performs various checks on the
      * rounding unit interval that is defined by this. Assumptions tested are
      * described in
diff --git a/server/src/test/java/org/opensearch/common/geo/GeoJsonShapeParserTests.java b/server/src/test/java/org/opensearch/common/geo/GeoJsonShapeParserTests.java
index 25140d1ef73d5..e16dd859ec14c 100644
--- a/server/src/test/java/org/opensearch/common/geo/GeoJsonShapeParserTests.java
+++ b/server/src/test/java/org/opensearch/common/geo/GeoJsonShapeParserTests.java
@@ -1169,9 +1169,9 @@ public void testParseOGCPolygonWithHoles() throws IOException, ParseException {
     }
 
     public void testParseInvalidPolygon() throws IOException {
-        /**
-         * The following 3 test cases ensure proper error handling of invalid polygons
-         * per the GeoJSON specification
+        /*
+          The following 3 test cases ensure proper error handling of invalid polygons
+          per the GeoJSON specification
          */
         // test case 1: create an invalid polygon with only 2 points
         String invalidPoly = XContentFactory.jsonBuilder()
diff --git a/server/src/test/java/org/opensearch/common/geo/GeometryIndexerTests.java b/server/src/test/java/org/opensearch/common/geo/GeometryIndexerTests.java
index 297b7d9f65d5f..7fc95c2316aef 100644
--- a/server/src/test/java/org/opensearch/common/geo/GeometryIndexerTests.java
+++ b/server/src/test/java/org/opensearch/common/geo/GeometryIndexerTests.java
@@ -225,7 +225,7 @@ public static MultiPoint remove180s(MultiPoint points) {
     /**
      * A randomized test that generates a random lines crossing anti-merdian and checks that the decomposed segments of this line
      * have the same total length (measured using Euclidean distances between neighboring points) as the original line.
-     *
+     * 
                            
      * It also extracts all points from these lines, performs normalization of these points and then compares that the resulting
      * points of line normalization match the points of points normalization with the exception of points that were created on the
      * antimeridian as the result of line decomposition.
diff --git a/server/src/test/java/org/opensearch/common/joda/JavaJodaTimeDuellingTests.java b/server/src/test/java/org/opensearch/common/joda/JavaJodaTimeDuellingTests.java
index 02bc6c58ad233..e8ddfde11f4cc 100644
--- a/server/src/test/java/org/opensearch/common/joda/JavaJodaTimeDuellingTests.java
+++ b/server/src/test/java/org/opensearch/common/joda/JavaJodaTimeDuellingTests.java
@@ -61,8 +61,8 @@ protected boolean enableWarningsCheck() {
     }
 
     public void testTimezoneParsing() {
-        /** this testcase won't work in joda. See comment in {@link #testPartialTimeParsing()}
-         *  assertSameDateAs("2016-11-30T+01", "strict_date_optional_time", "strict_date_optional_time");
+        /* this testcase won't work in joda. See comment in {@link #testPartialTimeParsing()}
+           assertSameDateAs("2016-11-30T+01", "strict_date_optional_time", "strict_date_optional_time");
          */
         assertSameDateAs("2016-11-30T00+01", "strict_date_optional_time", "strict_date_optional_time");
         assertSameDateAs("2016-11-30T00+0100", "strict_date_optional_time", "strict_date_optional_time");
diff --git a/server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java b/server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java
index 8297f8fcf47e2..d1b3adcd55f0c 100644
--- a/server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java
+++ b/server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java
@@ -212,7 +212,7 @@ public void testTimeUnitRoundingDST() {
      * {@link DateTimeUnit} and {@link DateTimeZone} and often (50% of the time)
      * chooses test dates that are exactly on or close to offset changes (e.g.
      * DST) in the chosen time zone.
-     *
+     * 
                            
      * It rounds the test date down and up and performs various checks on the
      * rounding unit interval that is defined by this. Assumptions tested are
      * described in
diff --git a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java
index f3f682697a930..9a0d34c916f5c 100644
--- a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java
@@ -1097,8 +1097,8 @@ public void testLimitNestedDocsMultipleNestedFields() throws Exception {
 
     @Override
     protected boolean forbidPrivateIndexSettings() {
-        /**
-         * This is needed to force the index version with {@link IndexMetadata.SETTING_INDEX_VERSION_CREATED}.
+        /*
+          This is needed to force the index version with {@link IndexMetadata.SETTING_INDEX_VERSION_CREATED}.
          */
         return false;
     }
diff --git a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java
index 729205c9775b4..7c7598e5dc3c4 100644
--- a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java
@@ -109,7 +109,7 @@ public void testSerialization() throws Exception {
     /**
      * Test that if we serialize and deserialize an object, further
      * serialization leads to identical bytes representation.
-     *
+     * 
                            
      * This is necessary to ensure because we use the serialized BytesReference
      * of this builder as part of the cacheKey in
      * {@link ShardSearchRequest} (via
diff --git a/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java b/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java
index 4bbb03322b1f8..2661873d9498f 100644
--- a/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java
+++ b/server/src/test/java/org/opensearch/search/CreatePitSingleNodeTests.java
@@ -321,9 +321,9 @@ public void testCreatePitMoreThanMaxOpenPitContexts() throws Exception {
         // deleteall
         DeletePitRequest deletePITRequest = new DeletePitRequest(pitIds.toArray(new String[0]));
 
-        /**
-         * When we invoke delete again, returns success after clearing the remaining readers. Asserting reader context
-         * not found exceptions don't result in failures ( as deletion in one node is successful )
+        /*
+          When we invoke delete again, returns success after clearing the remaining readers. Asserting reader context
+          not found exceptions don't result in failures ( as deletion in one node is successful )
          */
         ActionFuture execute = client().execute(DeletePitAction.INSTANCE, deletePITRequest);
         DeletePitResponse deletePITResponse = execute.get();
@@ -489,8 +489,8 @@ public void testPitAfterUpdateIndex() throws Exception {
                 client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get().getHits().getTotalHits().value,
                 Matchers.equalTo(50L)
             );
-            /**
-             * assert without point in time
+            /*
+              assert without point in time
              */
 
             assertThat(
@@ -509,8 +509,8 @@ public void testPitAfterUpdateIndex() throws Exception {
                 client().prepareSearch().setSize(0).setQuery(termQuery("message", "update")).get().getHits().getTotalHits().value,
                 Matchers.equalTo(50L)
             );
-            /**
-             * using point in time id will have the same search results as ones before update
+            /*
+              using point in time id will have the same search results as ones before update
              */
             assertThat(
                 client().prepareSearch()
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/DateScriptMocksPlugin.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/DateScriptMocksPlugin.java
index 5a9a9e2b6cb51..d6ba4eedd3a19 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/DateScriptMocksPlugin.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/DateScriptMocksPlugin.java
@@ -43,7 +43,7 @@
 
 /**
  * Mock scripts shared by DateRangeIT and DateHistogramIT.
- *
+ * 
                            
  * Provides {@link DateScriptMocksPlugin#EXTRACT_FIELD}, {@link DateScriptMocksPlugin#DOUBLE_PLUS_ONE_MONTH},
  * and {@link DateScriptMocksPlugin#LONG_PLUS_ONE_MONTH} scripts.
  */
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/LegacyIntervalCompositeAggBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/LegacyIntervalCompositeAggBuilderTests.java
index aa51f9b11ea19..39054f15826f0 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/LegacyIntervalCompositeAggBuilderTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/LegacyIntervalCompositeAggBuilderTests.java
@@ -44,7 +44,7 @@
 /**
  * Duplicates the tests from {@link CompositeAggregationBuilderTests}, except using the deprecated
  * interval on date histo.  Separated to make testing the warnings easier.
- *
+ * 
                            
  * Can be removed in when the legacy interval options are gone
  */
 public class LegacyIntervalCompositeAggBuilderTests extends BaseAggregationTestCase {
diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/MetricAggScriptPlugin.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/MetricAggScriptPlugin.java
index e29eb95a52cd7..b4dbc8017bc3b 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/metrics/MetricAggScriptPlugin.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/MetricAggScriptPlugin.java
@@ -45,7 +45,7 @@
 
 /**
  * Provides a number of dummy scripts for tests.
- *
+ * 
                            
  * Each script provided allows for an {@code inc} parameter which will
  * be added to each value read from a document.
  */
diff --git a/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java
index 4b8d0ef83dd1b..de213a154c3c5 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java
@@ -80,15 +80,15 @@ public class AvgBucketAggregatorTests extends AggregatorTestCase {
 
     /**
      * Test for issue #30608.  Under the following circumstances:
-     *
+     * 
                            
      * A. Multi-bucket agg in the first entry of our internal list
      * B. Regular agg as the immediate child of the multi-bucket in A
      * C. Regular agg with the same name as B at the top level, listed as the second entry in our internal list
      * D. Finally, a pipeline agg with the path down to B
-     *
+     * 
                            
      * BucketMetrics reduction would throw a class cast exception due to bad subpathing.  This test ensures
      * it is fixed.
-     *
+     * 
                            
      * Note: we have this test inside of the `avg_bucket` package so that we can get access to the package-private
      * `reduce()` needed for testing this
      */
diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java
index c19c1ebcb5c26..ce7344fa0d2d6 100644
--- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java
+++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineInfoTests.java
@@ -46,7 +46,7 @@ public void testSerializationRoundtrip() throws IOException {
 
     /**
      * When serializing / deserializing to / from old versions, processor type info is lost.
-     *
+     * 
                            
      * Also, we only supported request/response processors.
      */
     public void testSerializationRoundtripBackcompat() throws IOException {
diff --git a/test/framework/src/main/java/org/opensearch/action/support/ActionTestUtils.java b/test/framework/src/main/java/org/opensearch/action/support/ActionTestUtils.java
index 00fd5a5bcf815..52496c331c25e 100644
--- a/test/framework/src/main/java/org/opensearch/action/support/ActionTestUtils.java
+++ b/test/framework/src/main/java/org/opensearch/action/support/ActionTestUtils.java
@@ -55,7 +55,7 @@ public static  R
 
     /**
      * Executes the given action.
-     *
+     * 
                            
      * This is a shim method to make execution publicly available in tests.
      */
     public static  void execute(
diff --git a/test/framework/src/main/java/org/opensearch/cli/CommandTestCase.java b/test/framework/src/main/java/org/opensearch/cli/CommandTestCase.java
index 4ee1314d27fe1..5566b493adc7d 100644
--- a/test/framework/src/main/java/org/opensearch/cli/CommandTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/cli/CommandTestCase.java
@@ -54,7 +54,7 @@ public void resetTerminal() {
 
     /**
      * Runs a command with the given args.
-     *
+     * 
                            
      * Output can be found in {@link #terminal}.
      */
     public String execute(String... args) throws Exception {
diff --git a/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationWithConstraintsTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationWithConstraintsTestCase.java
index 7f1d1d3381751..0c08de252e4cd 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationWithConstraintsTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationWithConstraintsTestCase.java
@@ -209,28 +209,28 @@ public int allocateAndCheckIndexShardHotSpots(boolean expected, int nodes, Strin
                     continue;
                 }
 
-                /**
-                 * Hot spots can occur due to the order in which shards get allocated to nodes.
-                 * A node with fewer shards may not be able to accept current shard due to
-                 * SameShardAllocationDecider, causing it to breach allocation constraint on
-                 * another node. We need to differentiate between such hot spots v/s actual hot
-                 * spots.
-                 *
-                 * A simple check could be to ensure there is no node with shards less than
-                 * allocation limit, that can accept current shard. However, in current
-                 * allocation algorithm, when nodes get throttled, shards are added to
-                 * ModelNodes without adding them to actual cluster (RoutingNodes). As a result,
-                 * the shards per node we see here, are different from the ones observed by
-                 * weight function in balancer. RoutingNodes with {@link count} < {@link limit}
-                 * may not have had the same count in the corresponding ModelNode seen by weight
-                 * function. We hence use the following alternate check --
-                 *
-                 * Given the way {@link limit} is defined, we should not have hot spots if *all*
-                 * nodes are eligible to accept the shard. A hot spot is acceptable, if either
-                 * all peer nodes have {@link count} > {@link limit}, or if even one node is
-                 * ineligible to accept the shard due to SameShardAllocationDecider, as this
-                 * leads to a chain of events that breach IndexShardsPerNode constraint on all
-                 * other nodes.
+                /*
+                  Hot spots can occur due to the order in which shards get allocated to nodes.
+                  A node with fewer shards may not be able to accept current shard due to
+                  SameShardAllocationDecider, causing it to breach allocation constraint on
+                  another node. We need to differentiate between such hot spots v/s actual hot
+                  spots.
+
+                  A simple check could be to ensure there is no node with shards less than
+                  allocation limit, that can accept current shard. However, in current
+                  allocation algorithm, when nodes get throttled, shards are added to
+                  ModelNodes without adding them to actual cluster (RoutingNodes). As a result,
+                  the shards per node we see here, are different from the ones observed by
+                  weight function in balancer. RoutingNodes with {@link count} < {@link limit}
+                  may not have had the same count in the corresponding ModelNode seen by weight
+                  function. We hence use the following alternate check --
+
+                  Given the way {@link limit} is defined, we should not have hot spots if *all*
+                  nodes are eligible to accept the shard. A hot spot is acceptable, if either
+                  all peer nodes have {@link count} > {@link limit}, or if even one node is
+                  ineligible to accept the shard due to SameShardAllocationDecider, as this
+                  leads to a chain of events that breach IndexShardsPerNode constraint on all
+                  other nodes.
                  */
 
                 // If all peer nodes have count >= limit, hotspot is acceptable
diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java
index 60aacb83e0dc1..946b980bfb62f 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/LinearizabilityChecker.java
@@ -476,22 +476,22 @@ void unlift() {
 
     /**
      * A cache optimized for small bit-counts (less than 64) and small number of unique permutations of state objects.
-     *
+     * 
                            
      * Each combination of states is kept once only, building on the
      * assumption that the number of permutations is small compared to the
      * number of bits permutations. For those histories that are difficult to check
      * we will have many bits combinations that use the same state permutations.
-     *
+     * 
                            
      * The smallMap optimization allows us to avoid object overheads for bit-sets up to 64 bit large.
-     *
+     * 
                            
      * Comparing set of (bits, state) to smallMap:
      * (bits, state) : 24 (tuple) + 24 (FixedBitSet) + 24 (bits) + 5 (hash buckets) + 24 (hashmap node).
      * smallMap bits to {state} : 10 (bits) + 5 (hash buckets) + avg-size of unique permutations.
-     *
+     * 
                            
      * The avg-size of the unique permutations part is very small compared to the
      * sometimes large number of bits combinations (which are the cases where
      * we run into trouble).
-     *
+     * 
                            
      * set of (bits, state) totals 101 bytes compared to smallMap bits to { state }
      * which totals 15 bytes, ie. a 6x improvement in memory usage.
      */
diff --git a/test/framework/src/main/java/org/opensearch/common/logging/JsonLogsIntegTestCase.java b/test/framework/src/main/java/org/opensearch/common/logging/JsonLogsIntegTestCase.java
index e4774030f21cb..5f4d92d65548b 100644
--- a/test/framework/src/main/java/org/opensearch/common/logging/JsonLogsIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/common/logging/JsonLogsIntegTestCase.java
@@ -52,11 +52,11 @@
  * Fields available upon process startup: type, timestamp, level, component,
  * message, node.name, cluster.name.
  * Whereas node.id and cluster.uuid are available later once the first clusterState has been received.
- *
+ * 
                            
  *
  * node.name, cluster.name, node.id, cluster.uuid
  * should not change across all log lines
- *
+ * 
                            
  * Note that this won't pass for nodes in clusters that don't have the node name defined in opensearch.yml and start
  * with DEBUG or TRACE level logging. Those nodes log a few lines before the node.name is set by LogConfigurator.setNodeName.
  */
diff --git a/test/framework/src/main/java/org/opensearch/index/MockEngineFactoryPlugin.java b/test/framework/src/main/java/org/opensearch/index/MockEngineFactoryPlugin.java
index 72d34676850b9..0a47db4c740d6 100644
--- a/test/framework/src/main/java/org/opensearch/index/MockEngineFactoryPlugin.java
+++ b/test/framework/src/main/java/org/opensearch/index/MockEngineFactoryPlugin.java
@@ -46,7 +46,7 @@
 
 /**
  * A plugin to use {@link MockEngineFactory}.
- *
+ * 
                            
  * Subclasses may override the reader wrapper used.
  */
 public class MockEngineFactoryPlugin extends Plugin implements EnginePlugin {
diff --git a/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java
index f016d9450425d..5ab77783b2bac 100644
--- a/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java
+++ b/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java
@@ -56,7 +56,7 @@ private RandomCreateIndexGenerator() {}
 
     /**
      * Returns a random {@link CreateIndexRequest}.
-     *
+     * 
                            
      * Randomizes the index name, the aliases, mappings and settings associated with the
      * index. If present, the mapping definition will be nested under a type name.
      */
diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase.java
index 83bb2f1ee7d65..77137073aa30f 100644
--- a/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase.java
@@ -149,7 +149,7 @@ protected Set unsupportedProperties() {
 
     /**
      * Add type-specific modifiers for consistency checking.
-     *
+     * 
                            
      * This should be called in a {@code @Before} method
      */
     protected void addModifier(String property, boolean updateable, BiConsumer method) {
@@ -158,7 +158,7 @@ protected void addModifier(String property, boolean updateable, BiConsumer
 
     /**
      * Add type-specific modifiers for consistency checking.
-     *
+     * 
                            
      * This should be called in a {@code @Before} method
      */
     protected void addBooleanModifier(String property, boolean updateable, BiConsumer method) {
diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase2.java b/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase2.java
index 1987a49431bf9..5dfb2f16a1aae 100644
--- a/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase2.java
+++ b/test/framework/src/main/java/org/opensearch/index/mapper/FieldMapperTestCase2.java
@@ -139,7 +139,7 @@ protected Set unsupportedProperties() {
 
     /**
      * Add type-specific modifiers for consistency checking.
-     *
+     * 
                            
      * This should be called in a {@code @Before} method
      */
     protected void addModifier(String property, boolean updateable, BiConsumer method) {
@@ -148,7 +148,7 @@ protected void addModifier(String property, boolean updateable, BiConsumer
 
     /**
      * Add type-specific modifiers for consistency checking.
-     *
+     * 
                            
      * This should be called in a {@code @Before} method
      */
     protected void addBooleanModifier(String property, boolean updateable, BiConsumer method) {
diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
index ffa37817e8a03..9dc230474482f 100644
--- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
@@ -1086,7 +1086,7 @@ protected void recoverReplica(
     /**
      * Recovers a replica from the give primary, allow the user to supply a custom recovery target. A typical usage of a custom recovery
      * target is to assert things in the various stages of recovery.
-     *
+     * 
                            
      * Note: this method keeps the shard in {@link IndexShardState#POST_RECOVERY} and doesn't start it.
      *
      * @param replica                the recovery target shard
diff --git a/test/framework/src/main/java/org/opensearch/mockito/plugin/PriviledgedMockMaker.java b/test/framework/src/main/java/org/opensearch/mockito/plugin/PriviledgedMockMaker.java
index ef13e065968df..0f5e043ee1135 100644
--- a/test/framework/src/main/java/org/opensearch/mockito/plugin/PriviledgedMockMaker.java
+++ b/test/framework/src/main/java/org/opensearch/mockito/plugin/PriviledgedMockMaker.java
@@ -43,7 +43,7 @@ public class PriviledgedMockMaker implements MockMaker {
      * since Mockito does not support SecurityManager out of the box. The method has to be called by
      * test framework before the SecurityManager is being set, otherwise additional permissions have
      * to be granted to the caller:
-     *
+     * 
                            
      *     permission java.security.Permission "createAccessControlContext"
      *
      */
diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java
index bf020b1bc292d..dff9b997d87db 100644
--- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java
@@ -265,7 +265,7 @@ protected static void drainInputStream(final InputStream inputStream) throws IOE
 
     /**
      * HTTP handler that injects random service errors
-     *
+     * 
                            
      * Note: it is not a good idea to allow this handler to simulate too many errors as it would
      * slow down the test suite.
      */
@@ -339,7 +339,7 @@ public interface DelegatingHttpHandler extends HttpHandler {
 
     /**
      * HTTP handler that allows collect request stats per request type.
-     *
+     * 
                            
      * Implementors should keep track of the desired requests on {@link #maybeTrack(String, Headers)}.
      */
     @SuppressForbidden(reason = "this test uses a HttpServer to emulate a cloud-based storage service")
@@ -377,7 +377,7 @@ public void handle(HttpExchange exchange) throws IOException {
 
         /**
          * Tracks the given request if it matches the criteria.
-         *
+         * 
                            
          * The request is represented as:
          * Request = Method SP Request-URI
          *
diff --git a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
index cb0614ddeb808..83b245a1bcecb 100644
--- a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
@@ -58,14 +58,14 @@
 
 /**
  * A mocked script engine that can be used for testing purpose.
- *
+ * 
                            
  * This script engine allows to define a set of predefined scripts that basically a combination of a key and a
  * function:
- *
+ * 
                            
  * The key can be anything as long as it is a {@link String} and is used to resolve the scripts
  * at compilation time. For inline scripts, the key can be a description of the script. For stored and file scripts,
  * the source must match a key in the predefined set of scripts.
- *
+ * 
                            
  * The function is used to provide the result of the script execution and can return anything.
  */
 public class MockScriptEngine implements ScriptEngine {
diff --git a/test/framework/src/main/java/org/opensearch/script/ScoreAccessor.java b/test/framework/src/main/java/org/opensearch/script/ScoreAccessor.java
index 26b439fa6438f..9cf2141555957 100644
--- a/test/framework/src/main/java/org/opensearch/script/ScoreAccessor.java
+++ b/test/framework/src/main/java/org/opensearch/script/ScoreAccessor.java
@@ -39,7 +39,7 @@
 
 /**
  * A float encapsulation that dynamically accesses the score of a document.
- *
+ * 
                            
  * The provided {@link DocLookup} is used to retrieve the score
  * for the current document.
  */
diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
index 5c649f1dc832d..82f15a590bea6 100644
--- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
@@ -685,7 +685,7 @@ protected static IndexReader maybeWrapReaderEs(DirectoryReader reader) throws IO
      * Implementors should return a list of {@link ValuesSourceType} that the aggregator supports.
      * This is used to test the matrix of supported/unsupported field types against the aggregator
      * and verify it works (or doesn't) as expected.
-     *
+     * 
                            
      * If this method is implemented, {@link AggregatorTestCase#createAggBuilderForTypeTest(MappedFieldType, String)}
      * should be implemented as well.
      *
@@ -702,7 +702,7 @@ protected List getSupportedValuesSourceTypes() {
      * The field type and name are provided, and the implementor is expected to return an AggBuilder accordingly.
      * The AggBuilder should be returned even if the aggregation does not support the field type, because
      * the test will check if an exception is thrown in that case.
-     *
+     * 
                            
      * The list of supported types are provided by {@link AggregatorTestCase#getSupportedValuesSourceTypes()},
      * which must also be implemented.
      *
@@ -720,7 +720,7 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy
      * A method that allows implementors to specifically denylist particular field types (based on their content_name).
      * This is needed in some areas where the ValuesSourceType is not granular enough, for example integer values
      * vs floating points, or `keyword` bytes vs `binary` bytes (which are not searchable)
-     *
+     * 
                            
      * This is a denylist instead of an allowlist because there are vastly more field types than ValuesSourceTypes,
      * and it's expected that these unsupported cases are exceptional rather than common
      */
@@ -734,7 +734,7 @@ protected List unsupportedMappedFieldTypes() {
      * is provided by the implementor class, and it is executed against each field type in turn.  If
      * an exception is thrown when the field is supported, that will fail the test.  Similarly, if
      * an exception _is not_ thrown when a field is unsupported, that will also fail the test.
-     *
+     * 
                            
      * Exception types/messages are not currently checked, just presence/absence of an exception.
      */
     public void testSupportedFieldTypes() throws IOException {
@@ -825,7 +825,7 @@ private ValuesSourceType fieldToVST(MappedFieldType fieldType) {
 
     /**
      * Helper method to write a single document with a single value specific to the requested fieldType.
-     *
+     * 
                            
      * Throws an exception if it encounters an unknown field type, to prevent new ones from sneaking in without
      * being tested.
      */
diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java
index f5d358a162bd1..10e782e6af8da 100644
--- a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableSerializationTestCase.java
@@ -41,8 +41,8 @@
 
 /**
  * An abstract test case to ensure correct behavior of Diffable.
- *
- * This class can be used as a based class for tests of Metadata.Custom classes and other classes that support,
+ * 
                            
+ * This class can be used as a based class for tests of Metadata.Custom classes and other classes that support {@link org.opensearch.core.common.io.stream.Writeable } serialization, XContent-based serialization and is diffable.
  * Writable serialization, XContent-based serialization and is diffable.
  */
 public abstract class AbstractDiffableSerializationTestCase & ToXContent> extends AbstractSerializingTestCase {
diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java
index ff7c39cd8f102..3f97f0704d733 100644
--- a/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/AbstractDiffableWireSerializationTestCase.java
@@ -40,8 +40,8 @@
 
 /**
  * An abstract test case to ensure correct behavior of Diffable.
- *
- * This class can be used as a based class for tests of ClusterState.Custom classes and other classes that support,
+ * 
                            
+ * This class can be used as a based class for tests of ClusterState.Custom classes and other classes that support {@link org.opensearch.core.common.io.stream.Writeable } serialization and is diffable.
  * Writable serialization and is diffable.
  */
 public abstract class AbstractDiffableWireSerializationTestCase> extends AbstractWireSerializingTestCase {
diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java
index 47fe85d28975f..afd93e1b72fbb 100644
--- a/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/AbstractQueryTestCase.java
@@ -213,7 +213,7 @@ public void testUnknownObjectException() throws IOException {
     /**
      * Traverses the json tree of the valid query provided as argument and mutates it one or more times by adding one object within each
      * object encountered.
-     *
+     * 
                            
      * For instance given the following valid term query:
      * {
      *     "term" : {
@@ -222,7 +222,7 @@ public void testUnknownObjectException() throws IOException {
      *         }
      *     }
      * }
-     *
+     * 
                            
      * The following two mutations will be generated, and an exception is expected when trying to parse them:
      * {
      *     "term" : {
@@ -233,7 +233,7 @@ public void testUnknownObjectException() throws IOException {
      *         }
      *     }
      * }
-     *
+     * 
                            
      * {
      *     "term" : {
      *         "field" : {
@@ -243,7 +243,7 @@ public void testUnknownObjectException() throws IOException {
      *         }
      *     }
      * }
-     *
+     * 
                            
      * Every mutation is then added to the list of results with a boolean flag indicating if a parsing exception is expected or not
      * for the mutation. Some specific objects do not cause any exception as they can hold arbitrary content; they are passed using the
      * arbitraryMarkers parameter.
@@ -768,7 +768,7 @@ protected static String randomMinimumShouldMatch() {
     /**
      * Call this method to check a valid json string representing the query under test against
      * it's generated json.
-     *
+     * 
                            
      * Note: By the time of this writing (Nov 2015) all queries are taken from the query dsl
      * reference docs mirroring examples there. Here's how the queries were generated:
      *
diff --git a/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java b/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java
index 9a4363dc4d946..64c5d916d55d2 100644
--- a/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/AbstractWireTestCase.java
@@ -124,9 +124,9 @@ protected final T copyInstance(T instance) throws IOException {
 
     /**
      * Get the {@link NamedWriteableRegistry} to use when de-serializing the object.
-     *
+     * 
                            
      * Override this method if you need to register {@link NamedWriteable}s for the test object to de-serialize.
-     *
+     * 
                            
      * By default this will return a {@link NamedWriteableRegistry} with no registered {@link NamedWriteable}s
      */
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
diff --git a/test/framework/src/main/java/org/opensearch/test/MockKeywordPlugin.java b/test/framework/src/main/java/org/opensearch/test/MockKeywordPlugin.java
index c27f3f169fbae..3d829d77dd323 100644
--- a/test/framework/src/main/java/org/opensearch/test/MockKeywordPlugin.java
+++ b/test/framework/src/main/java/org/opensearch/test/MockKeywordPlugin.java
@@ -44,7 +44,7 @@
 /**
  * Some tests rely on the keyword tokenizer, but this tokenizer isn't part of lucene-core and therefor not available
  * in some modules. What this test plugin does, is use the mock tokenizer and advertise that as the keyword tokenizer.
- *
+ * 
                            
  * Most tests that need this test plugin use normalizers. When normalizers are constructed they try to resolve the
  * keyword tokenizer, but if the keyword tokenizer isn't available then constructing normalizers will fail.
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index 0b80c6e577f95..da829a3bc5225 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -1343,7 +1343,7 @@ protected void ensureStableCluster(int nodeCount, TimeValue timeValue, boolean l
 
     /**
      * Ensures that all nodes in the cluster are connected to each other.
-     *
+     * 
                            
      * Some network disruptions may leave nodes that are not the cluster-manager disconnected from each other.
      * {@link org.opensearch.cluster.NodeConnectionsService} will eventually reconnect but it's
      * handy to be able to ensure this happens faster
@@ -2311,11 +2311,11 @@ public static void afterClass() throws Exception {
 
     private static void initializeSuiteScope() throws Exception {
         Class targetClass = getTestClass();
-        /**
-         * Note we create these test class instance via reflection
-         * since JUnit creates a new instance per test and that is also
-         * the reason why INSTANCE is static since this entire method
-         * must be executed in a static context.
+        /*
+          Note we create these test class instance via reflection
+          since JUnit creates a new instance per test and that is also
+          the reason why INSTANCE is static since this entire method
+          must be executed in a static context.
          */
         assert INSTANCE == null;
         if (isSuiteScopedTest(targetClass)) {
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTokenStreamTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTokenStreamTestCase.java
index e853c1e6314e1..9f7f33b351be3 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTokenStreamTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTokenStreamTestCase.java
@@ -48,9 +48,9 @@
 @TimeoutSuite(millis = TimeUnits.HOUR)
 @LuceneTestCase.SuppressReproduceLine
 @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
-/**
- * Basic test case for token streams. the assertion methods in this class will
- * run basic checks to enforce correct behavior of the token streams.
+/*
+  Basic test case for token streams. the assertion methods in this class will
+  run basic checks to enforce correct behavior of the token streams.
  */
 public abstract class OpenSearchTokenStreamTestCase extends BaseTokenStreamTestCase {
 
diff --git a/test/framework/src/main/java/org/opensearch/test/TestCluster.java b/test/framework/src/main/java/org/opensearch/test/TestCluster.java
index a9d9abf69ee41..8c41e6e5d5b38 100644
--- a/test/framework/src/main/java/org/opensearch/test/TestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/TestCluster.java
@@ -276,7 +276,7 @@ public void wipeRepositories(String... repositories) {
 
     /**
      * Ensures that any breaker statistics are reset to 0.
-     *
+     * 
                            
      * The implementation is specific to the test cluster, because the act of
      * checking some breaker stats can increase them.
      */
diff --git a/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java
index 0cb39a29e8159..2c535211ca456 100644
--- a/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java
+++ b/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java
@@ -164,10 +164,10 @@ private static String differenceBetweenObjectsIgnoringArrayOrder(String path, Ob
      * This method takes the input xContent data and adds a random field value, inner object or array into each
      * json object. This can e.g. be used to test if parsers that handle the resulting xContent can handle the
      * augmented xContent correctly, for example when testing lenient parsing.
-     *
+     * 
                            
      * If the xContent output contains objects that should be skipped of such treatment, an optional filtering
      * {@link Predicate} can be supplied that checks xContent paths that should be excluded from this treatment.
-     *
+     * 
                            
      * This predicate should check the xContent path that we want to insert to and return {@code true} if the
      * path should be excluded. Paths are string concatenating field names and array indices, so e.g. in:
      *
@@ -188,7 +188,7 @@ private static String differenceBetweenObjectsIgnoringArrayOrder(String path, Ob
      * 
      *
      * "foo1.bar.2.baz" would point to the desired insert location.
-     *
+     * 
                            
      * To exclude inserting into the "foo1" object we would user a {@link Predicate} like
      * 
                                  * {@code
@@ -257,12 +257,12 @@ public static BytesReference insertRandomFields(
      * This utility method takes an XContentParser and walks the xContent structure to find all
      * possible paths to where a new object or array starts. This can be used in tests that add random
      * xContent values to test parsing code for errors or to check their robustness against new fields.
-     *
+     * 
                            
      * The path uses dot separated fieldnames and numbers for array indices, similar to what we do in
      * {@link ObjectPath}.
-     *
+     * 
                            
      * The {@link Stack} passed in should initially be empty, it gets pushed to by recursive calls
-     *
+     * 
                            
      * As an example, the following json xContent:
      * 
                                  *     {
diff --git a/test/framework/src/main/java/org/opensearch/test/client/NoOpClient.java b/test/framework/src/main/java/org/opensearch/test/client/NoOpClient.java
index a0e87d5fd7189..45d779b3e8697 100644
--- a/test/framework/src/main/java/org/opensearch/test/client/NoOpClient.java
+++ b/test/framework/src/main/java/org/opensearch/test/client/NoOpClient.java
@@ -47,7 +47,7 @@
 /**
  * Client that always responds with {@code null} to every request. Override {@link #doExecute(ActionType, ActionRequest, ActionListener)}
  * for testing.
- *
+ * 
                            
  * See also {@link NoOpNodeClient} if you need to mock a {@link org.opensearch.client.node.NodeClient}.
  */
 public class NoOpClient extends AbstractClient {
diff --git a/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java b/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java
index 7dfe9298e9a92..4e84fe3b91d15 100644
--- a/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java
+++ b/test/framework/src/main/java/org/opensearch/test/client/NoOpNodeClient.java
@@ -55,7 +55,7 @@
  * Client that always response with {@code null} to every request. Override {@link #doExecute(ActionType, ActionRequest, ActionListener)},
  * {@link #executeLocally(ActionType, ActionRequest, ActionListener)}, or {@link #executeLocally(ActionType, ActionRequest, TaskListener)}
  * for testing.
- *
+ * 
                            
  * See also {@link NoOpClient} if you do not specifically need a {@link NodeClient}.
  */
 public class NoOpNodeClient extends NodeClient {
diff --git a/test/framework/src/main/java/org/opensearch/test/disruption/LongGCDisruption.java b/test/framework/src/main/java/org/opensearch/test/disruption/LongGCDisruption.java
index 690e15dd80873..44837c37962b4 100644
--- a/test/framework/src/main/java/org/opensearch/test/disruption/LongGCDisruption.java
+++ b/test/framework/src/main/java/org/opensearch/test/disruption/LongGCDisruption.java
@@ -248,7 +248,7 @@ public TimeValue expectedTimeToHeal() {
     /**
      * resolves all threads belonging to given node and suspends them if their current stack trace
      * is "safe". Threads are added to nodeThreads if suspended.
-     *
+     * 
                            
      * returns true if some live threads were found. The caller is expected to call this method
      * until no more "live" are found.
      */
diff --git a/test/framework/src/main/java/org/opensearch/test/disruption/NetworkDisruption.java b/test/framework/src/main/java/org/opensearch/test/disruption/NetworkDisruption.java
index e77b8f5b24897..62e19750a363b 100644
--- a/test/framework/src/main/java/org/opensearch/test/disruption/NetworkDisruption.java
+++ b/test/framework/src/main/java/org/opensearch/test/disruption/NetworkDisruption.java
@@ -109,7 +109,7 @@ public void ensureHealthy(InternalTestCluster cluster) {
 
     /**
      * Ensures that all nodes in the cluster are connected to each other.
-     *
+     * 
                            
      * Some network disruptions may leave nodes that are not the cluster-manager disconnected from each other.
      * {@link org.opensearch.cluster.NodeConnectionsService} will eventually reconnect but it's
      * handy to be able to ensure this happens faster
diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java
index 107e42ce43487..7462062a0cd46 100644
--- a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java
+++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java
@@ -57,13 +57,13 @@
  * A gateway allocator implementation that keeps an in memory list of started shard allocation
  * that are used as replies to the, normally async, fetch data requests. The in memory list
  * is adapted when shards are started and failed.
- *
+ * 
                            
  * Nodes leaving and joining the cluster do not change the list of shards the class tracks but
  * rather serves as a filter to what is returned by fetch data. Concretely - fetch data will
  * only return shards that were started on nodes that are currently part of the cluster.
- *
+ * 
                            
  * For now only primary shard related data is fetched. Replica request always get an empty response.
- *
+ * 
                            
  *
  * This class is useful to use in unit tests that require the functionality of {@link GatewayAllocator} but do
  * not have all the infrastructure required to use it.
diff --git a/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestIssueLogging.java b/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestIssueLogging.java
index f81c47b6a0a08..97fdcb796f195 100644
--- a/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestIssueLogging.java
+++ b/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestIssueLogging.java
@@ -43,7 +43,7 @@
 /**
  * Annotation used to set a custom log level when investigating test failures. Do not use this annotation to explicitly
  * control the logging level in tests; instead, use {@link TestLogging}.
- *
+ * 
                            
  * It supports multiple logger:level comma-separated key-value pairs of logger:level (e.g.,
  * org.opensearch.cluster.metadata:TRACE). Use the _root keyword to set the root logger level.
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestLogging.java b/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestLogging.java
index d440fae409897..6c0b87ac67354 100644
--- a/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestLogging.java
+++ b/test/framework/src/main/java/org/opensearch/test/junit/annotations/TestLogging.java
@@ -43,7 +43,7 @@
 /**
  * Annotation used to set a custom log level for controlling logging behavior in tests. Do not use this annotation when
  * investigating test failures; instead, use {@link TestIssueLogging}.
- *
+ * 
                            
  * It supports multiple logger:level comma-separated key-value pairs of logger:level (e.g.,
  * org.opensearch.cluster.metadata:TRACE). Use the _root keyword to set the root logger level.
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/junit/listeners/LoggingListener.java b/test/framework/src/main/java/org/opensearch/test/junit/listeners/LoggingListener.java
index 7c94c16b77471..ea2c5d055ed8b 100644
--- a/test/framework/src/main/java/org/opensearch/test/junit/listeners/LoggingListener.java
+++ b/test/framework/src/main/java/org/opensearch/test/junit/listeners/LoggingListener.java
@@ -53,7 +53,7 @@
  * A {@link RunListener} that allows changing the log level for a specific test method. When a test method is annotated with the
  * {@link TestLogging} annotation, the level for the specified loggers will be internally saved before the test method execution and
  * overridden with the specified ones. At the end of the test method execution the original loggers levels will be restored.
- *
+ * 
                            
  * This class is not thread-safe. Given the static nature of the logging API, it assumes that tests are never run concurrently in the same
  * JVM. For the very same reason no synchronization has been implemented regarding the save/restore process of the original loggers
  * levels.
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java
index dc13924195254..a77865579f3b3 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java
@@ -103,7 +103,7 @@ protected void dispatchRequest(RestRequest request) {
     /**
      * A mocked {@link NodeClient} which can be easily reconfigured to verify arbitrary verification
      * functions, and can be reset to allow reconfiguration partway through a test without having to construct a new object.
-     *
+     * 
                            
      * By default, will throw {@link AssertionError} when any execution method is called, unless configured otherwise using
      * {@link #setExecuteVerifier(BiFunction)} or {@link #setExecuteLocallyVerifier(BiFunction)}.
      */
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/DenylistedPathPatternMatcher.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/DenylistedPathPatternMatcher.java
index eeaa76b6ca1b3..3a80f25c5d4ab 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/DenylistedPathPatternMatcher.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/DenylistedPathPatternMatcher.java
@@ -35,7 +35,7 @@
 
 /**
  * Matches denylist patterns.
- *
+ * 
                            
  * Currently the following syntax is supported:
  *
  * 
                              
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/Features.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/Features.java
index 10fb1e52259a9..8e0bc03b08442 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/Features.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/Features.java
@@ -41,7 +41,7 @@
  * Allows to register additional features supported by the tests runner.
  * This way any runner can add extra features and use proper skip sections to avoid
  * breaking others runners till they have implemented the new feature as well.
- *
+ * 
                              
  * Once all runners have implemented the feature, it can be removed from the list
  * and the related skip sections can be removed from the tests as well.
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java
index c3289e7651056..12599f003077d 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/OpenSearchClientYamlSuiteTestCase.java
@@ -78,7 +78,7 @@
 /**
  * Runs a suite of yaml tests shared with all the official OpenSearch
  * clients against an opensearch cluster.
- *
+ * 
                              
  * The suite timeout is extended to account for projects with a large number of tests.
  */
 @TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
@@ -110,9 +110,9 @@ public abstract class OpenSearchClientYamlSuiteTestCase extends OpenSearchRestTe
     /**
      * This separator pattern matches ',' except it is preceded by a '\'.
      * This allows us to support ',' within paths when it is escaped with a slash.
-     *
+     * 
                              
      * For example, the path string "/a/b/c\,d/e/f,/foo/bar,/baz" is separated to "/a/b/c\,d/e/f", "/foo/bar" and "/baz".
-     *
+     * 
                              
      * For reference, this regular expression feature is known as zero-width negative look-behind.
      *
      */
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java
index ef8bf37d88227..ae1c33b1eb47f 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/DoSection.java
@@ -81,7 +81,7 @@
 
 /**
  * Represents a do section:
- *
+ * 
                              
  *   - do:
  *      catch:      missing
  *      headers:
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
index 2132c2ebab51c..8e929eff44348 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
@@ -47,7 +47,7 @@
 
 /**
  * Represents a gte assert section:
- *
+ * 
                              
  *   - gte:     { fields._ttl: 0 }
  */
 public class GreaterThanEqualToAssertion extends Assertion {
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsFalseAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsFalseAssertion.java
index 6cfbbfc5df8d5..999486ad04455 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsFalseAssertion.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsFalseAssertion.java
@@ -45,7 +45,7 @@
 
 /**
  * Represents an is_false assert section:
- *
+ * 
                              
  *   - is_false:  get.fields.bar
  *
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsTrueAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsTrueAssertion.java
index e746542d89126..bf5822406f014 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsTrueAssertion.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/IsTrueAssertion.java
@@ -46,7 +46,7 @@
 
 /**
  * Represents an is_true assert section:
- *
+ * 
                              
  *   - is_true:  get.fields.bar
  *
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanAssertion.java
index 263e0c8fb9c42..d6e2ae1e23996 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanAssertion.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanAssertion.java
@@ -46,7 +46,7 @@
 
 /**
  * Represents a lt assert section:
- *
+ * 
                              
  *  - lt:    { fields._ttl: 20000}
  *
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
index f0e7d0c01b8f0..ee46c04496f32 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
@@ -47,7 +47,7 @@
 
 /**
  * Represents a lte assert section:
- *
+ * 
                              
  *   - lte:     { fields._ttl: 0 }
  */
 public class LessThanOrEqualToAssertion extends Assertion {
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java
index a97e3e4cb77ed..77d8f3154729e 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java
@@ -50,7 +50,7 @@
 
 /**
  * Represents a match assert section:
- *
+ * 
                              
  *   - match:   { get.fields._routing: "5" }
  *
  */
diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java
index a561a53119a96..c8004d9807cb9 100644
--- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java
+++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/SetSection.java
@@ -42,7 +42,7 @@
 
 /**
  * Represents a set section:
- *
+ * 
                              
  *   - set: {_scroll_id: scroll_id}
  *
  */

From ed1b624481eb9c0cc2fd941b54eb0243807b399d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 11 Oct 2023 07:31:44 -0400
Subject: [PATCH 145/388] Bump de.thetaphi:forbiddenapis from 3.5.1 to 3.6 in
 /buildSrc (#10508)

* Bump de.thetaphi:forbiddenapis from 3.5.1 to 3.6 in /buildSrc

Bumps de.thetaphi:forbiddenapis from 3.5.1 to 3.6.

---
updated-dependencies:
- dependency-name: de.thetaphi:forbiddenapis
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 CHANGELOG.md          | 1 +
 buildSrc/build.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 467e019270a47..572a8346a0686 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -122,6 +122,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210))
 - Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
+- Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
 
 ### Changed
 - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index ccf6a6010ff63..6d3e0f018657e 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -114,7 +114,7 @@ dependencies {
   api 'com.github.johnrengelman:shadow:8.1.1'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
-  api 'de.thetaphi:forbiddenapis:3.5.1'
+  api 'de.thetaphi:forbiddenapis:3.6'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.4'

From 9bcd7eac2b3fde6953f74ea3ecfcb1a6a51404d3 Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Wed, 11 Oct 2023 18:38:08 +0530
Subject: [PATCH 146/388] [S3 Repository] Make bulk delete size configurable
 (#10445)

Signed-off-by: Gaurav Bafna 
---
 .../opensearch/repositories/s3/S3BlobContainer.java   | 11 ++---------
 .../org/opensearch/repositories/s3/S3BlobStore.java   | 10 ++++++++++
 .../org/opensearch/repositories/s3/S3Repository.java  | 11 +++++++++++
 .../s3/S3BlobContainerMockClientTests.java            |  2 ++
 .../repositories/s3/S3BlobContainerRetriesTests.java  |  2 ++
 .../repositories/s3/S3BlobStoreContainerTests.java    |  7 +++++++
 6 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
index 9ffdba5eaae3a..9777bd974d56c 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
@@ -113,13 +113,6 @@ class S3BlobContainer extends AbstractBlobContainer implements AsyncMultiStreamB
 
     private static final Logger logger = LogManager.getLogger(S3BlobContainer.class);
 
-    /**
-     * Maximum number of deletes in a {@link DeleteObjectsRequest}.
-     *
-     * @see S3 Documentation.
-     */
-    private static final int MAX_BULK_DELETES = 1000;
-
     private final S3BlobStore blobStore;
     private final String keyPath;
 
@@ -339,12 +332,12 @@ private void doDeleteBlobs(List blobNames, boolean relative) throws IOEx
             outstanding = new HashSet<>(blobNames);
         }
         try (AmazonS3Reference clientReference = blobStore.clientReference()) {
-            // S3 API only allows 1k blobs per delete so we split up the given blobs into requests of max. 1k deletes
+            // S3 API allows 1k blobs per delete so we split up the given blobs into requests of bulk size deletes
             final List deleteRequests = new ArrayList<>();
             final List partition = new ArrayList<>();
             for (String key : outstanding) {
                 partition.add(key);
-                if (partition.size() == MAX_BULK_DELETES) {
+                if (partition.size() == blobStore.getBulkDeletesSize()) {
                     deleteRequests.add(bulkDelete(blobStore.bucket(), partition));
                     partition.clear();
                 }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
index 3dd373b5b9f32..80005d92344a4 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
@@ -52,6 +52,7 @@
 
 import static org.opensearch.repositories.s3.S3Repository.BUCKET_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.BUFFER_SIZE_SETTING;
+import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
 import static org.opensearch.repositories.s3.S3Repository.CANNED_ACL_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING;
@@ -74,6 +75,8 @@ class S3BlobStore implements BlobStore {
 
     private volatile StorageClass storageClass;
 
+    private volatile int bulkDeletesSize;
+
     private volatile RepositoryMetadata repositoryMetadata;
 
     private final StatsMetricPublisher statsMetricPublisher = new StatsMetricPublisher();
@@ -92,6 +95,7 @@ class S3BlobStore implements BlobStore {
         ByteSizeValue bufferSize,
         String cannedACL,
         String storageClass,
+        int bulkDeletesSize,
         RepositoryMetadata repositoryMetadata,
         AsyncTransferManager asyncTransferManager,
         AsyncExecutorContainer priorityExecutorBuilder,
@@ -105,6 +109,7 @@ class S3BlobStore implements BlobStore {
         this.bufferSize = bufferSize;
         this.cannedACL = initCannedACL(cannedACL);
         this.storageClass = initStorageClass(storageClass);
+        this.bulkDeletesSize = bulkDeletesSize;
         this.repositoryMetadata = repositoryMetadata;
         this.asyncTransferManager = asyncTransferManager;
         this.normalExecutorBuilder = normalExecutorBuilder;
@@ -119,6 +124,7 @@ public void reload(RepositoryMetadata repositoryMetadata) {
         this.bufferSize = BUFFER_SIZE_SETTING.get(repositoryMetadata.settings());
         this.cannedACL = initCannedACL(CANNED_ACL_SETTING.get(repositoryMetadata.settings()));
         this.storageClass = initStorageClass(STORAGE_CLASS_SETTING.get(repositoryMetadata.settings()));
+        this.bulkDeletesSize = BULK_DELETE_SIZE.get(repositoryMetadata.settings());
     }
 
     @Override
@@ -150,6 +156,10 @@ public long bufferSizeInBytes() {
         return bufferSize.getBytes();
     }
 
+    public int getBulkDeletesSize() {
+        return bulkDeletesSize;
+    }
+
     @Override
     public BlobContainer blobContainer(BlobPath path) {
         return new S3BlobContainer(path, this);
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
index bbc3451be1535..aaf5b79891cdc 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
@@ -190,6 +190,13 @@ class S3Repository extends MeteredBlobStoreRepository {
         new ByteSizeValue(5, ByteSizeUnit.TB)
     );
 
+    /**
+     * Maximum number of deletes in a DeleteObjectsRequest.
+     *
+     * @see S3 Documentation.
+     */
+    static final Setting BULK_DELETE_SIZE = Setting.intSetting("bulk_delete_size", 1000, 1, 1000);
+
     /**
      * Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy,
      * standard_ia, onezone_ia and intelligent_tiering. Defaults to standard.
@@ -231,6 +238,8 @@ class S3Repository extends MeteredBlobStoreRepository {
     private final AsyncExecutorContainer normalExecutorBuilder;
     private final Path pluginConfigPath;
 
+    private volatile int bulkDeletesSize;
+
     // Used by test classes
     S3Repository(
         final RepositoryMetadata metadata,
@@ -340,6 +349,7 @@ protected S3BlobStore createBlobStore() {
             bufferSize,
             cannedACL,
             storageClass,
+            bulkDeletesSize,
             metadata,
             asyncUploadUtils,
             priorityExecutorBuilder,
@@ -399,6 +409,7 @@ private void readRepositoryMetadata() {
         this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());
         this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings());
         this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings());
+        this.bulkDeletesSize = BULK_DELETE_SIZE.get(metadata.settings());
         if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) {
             // provided repository settings
             deprecationLogger.deprecate(
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
index 8c8524212e08e..6eb8faa746d34 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
@@ -64,6 +64,7 @@
 
 import org.mockito.invocation.InvocationOnMock;
 
+import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -387,6 +388,7 @@ private S3BlobStore createBlobStore() {
             S3Repository.BUFFER_SIZE_SETTING.getDefault(Settings.EMPTY),
             S3Repository.CANNED_ACL_SETTING.getDefault(Settings.EMPTY),
             S3Repository.STORAGE_CLASS_SETTING.getDefault(Settings.EMPTY),
+            BULK_DELETE_SIZE.get(Settings.EMPTY),
             repositoryMetadata,
             new AsyncTransferManager(
                 S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
index ecad68474b601..a2214f5218991 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -95,6 +95,7 @@
 import static org.opensearch.repositories.s3.S3ClientSettings.MAX_RETRIES_SETTING;
 import static org.opensearch.repositories.s3.S3ClientSettings.READ_TIMEOUT_SETTING;
 import static org.opensearch.repositories.s3.S3ClientSettings.REGION;
+import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -215,6 +216,7 @@ protected AsyncMultiStreamBlobContainer createBlobContainer(
                 bufferSize == null ? S3Repository.BUFFER_SIZE_SETTING.getDefault(Settings.EMPTY) : bufferSize,
                 S3Repository.CANNED_ACL_SETTING.getDefault(Settings.EMPTY),
                 S3Repository.STORAGE_CLASS_SETTING.getDefault(Settings.EMPTY),
+                BULK_DELETE_SIZE.get(Settings.EMPTY),
                 repositoryMetadata,
                 new AsyncTransferManager(
                     S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
index e266bba372d80..2701cae6a733b 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
@@ -276,10 +276,12 @@ public void testDelete() throws IOException {
         final String bucketName = randomAlphaOfLengthBetween(1, 10);
 
         final BlobPath blobPath = new BlobPath();
+        int bulkDeleteSize = 5;
 
         final S3BlobStore blobStore = mock(S3BlobStore.class);
         when(blobStore.bucket()).thenReturn(bucketName);
         when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
+        when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize);
 
         final S3Client client = mock(S3Client.class);
         doAnswer(invocation -> new AmazonS3Reference(client)).when(blobStore).clientReference();
@@ -297,8 +299,11 @@ public void testDelete() throws IOException {
         when(client.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Iterable);
 
         final List keysDeleted = new ArrayList<>();
+        AtomicInteger deleteCount = new AtomicInteger();
         doAnswer(invocation -> {
             DeleteObjectsRequest deleteObjectsRequest = invocation.getArgument(0);
+            deleteCount.getAndIncrement();
+            logger.info("Object sizes are{}", deleteObjectsRequest.delete().objects().size());
             keysDeleted.addAll(deleteObjectsRequest.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()));
             return DeleteObjectsResponse.builder().build();
         }).when(client).deleteObjects(any(DeleteObjectsRequest.class));
@@ -311,6 +316,8 @@ public void testDelete() throws IOException {
         // keysDeleted will have blobPath also
         assertEquals(listObjectsV2ResponseIterator.getKeysListed().size(), keysDeleted.size() - 1);
         assertTrue(keysDeleted.contains(blobPath.buildAsString()));
+        // keysDeleted will have blobPath also
+        assertEquals((int) Math.ceil(((double) keysDeleted.size() + 1) / bulkDeleteSize), deleteCount.get());
         keysDeleted.remove(blobPath.buildAsString());
         assertEquals(new HashSet<>(listObjectsV2ResponseIterator.getKeysListed()), new HashSet<>(keysDeleted));
     }

From c4f5dd9f4d45530ad22f165f5ec88736534e5827 Mon Sep 17 00:00:00 2001
From: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com>
Date: Wed, 11 Oct 2023 07:31:18 -0700
Subject: [PATCH 147/388] Fix flaky query profile phase tests with concurrent
 search enabled (#10547) (#10547)

Signed-off-by: Ticheng Lin 
---
 CHANGELOG.md                                  |  2 +-
 .../search/query/QueryProfilePhaseTests.java  | 22 +++++--------------
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 572a8346a0686..2e107d2c04539 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -158,4 +158,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x
\ No newline at end of file
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x
diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java
index 92d27032b62e3..28af8a63cfba8 100644
--- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java
@@ -1066,10 +1066,6 @@ public void testIndexSortScrollOptimization() throws Exception {
     }
 
     public void testDisableTopScoreCollection() throws Exception {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10469",
-            executor != null
-        );
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer());
         RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
@@ -1108,10 +1104,10 @@ public void testDisableTopScoreCollection() throws Exception {
             assertThat(query.getTimeBreakdown().get("score_count"), greaterThan(0L));
             if (executor != null) {
                 assertThat(query.getTimeBreakdown().get("max_score"), greaterThan(0L));
-                assertThat(query.getTimeBreakdown().get("min_score"), greaterThan(0L));
+                assertThat(query.getTimeBreakdown().get("min_score"), greaterThanOrEqualTo(0L));
                 assertThat(query.getTimeBreakdown().get("avg_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("max_score_count"), greaterThan(0L));
-                assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThan(0L));
+                assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThanOrEqualTo(0L));
                 assertThat(query.getTimeBreakdown().get("avg_score_count"), greaterThan(0L));
             }
             assertThat(query.getTimeBreakdown().get("create_weight"), greaterThan(0L));
@@ -1241,10 +1237,6 @@ public void testMinScore() throws Exception {
     }
 
     public void testMaxScore() throws Exception {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/9932",
-            executor != null
-        );
         Directory dir = newDirectory();
         final Sort sort = new Sort(new SortField("filter", SortField.Type.STRING));
         IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort);
@@ -1286,7 +1278,7 @@ public void testMaxScore() throws Exception {
                 assertThat(query.getTimeBreakdown().get("max_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("min_score"), greaterThanOrEqualTo(0L));
                 assertThat(query.getTimeBreakdown().get("avg_score"), greaterThan(0L));
-                assertThat(query.getTimeBreakdown().get("max_score_count"), greaterThanOrEqualTo(6L));
+                assertThat(query.getTimeBreakdown().get("max_score_count"), greaterThanOrEqualTo(4L));
                 assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThanOrEqualTo(0L));
                 assertThat(query.getTimeBreakdown().get("avg_score_count"), greaterThanOrEqualTo(1L));
             }
@@ -1364,10 +1356,6 @@ public void testMaxScore() throws Exception {
     }
 
     public void testCollapseQuerySearchResults() throws Exception {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10139",
-            executor != null
-        );
         Directory dir = newDirectory();
         final Sort sort = new Sort(new SortField("user", SortField.Type.INT));
         IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort);
@@ -1412,7 +1400,7 @@ public void testCollapseQuerySearchResults() throws Exception {
                 assertThat(query.getTimeBreakdown().get("min_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("avg_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("max_score_count"), greaterThanOrEqualTo(6L));
-                assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThanOrEqualTo(6L));
+                assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThanOrEqualTo(2L));
                 assertThat(query.getTimeBreakdown().get("avg_score_count"), greaterThanOrEqualTo(6L));
             }
             assertThat(query.getTimeBreakdown().get("create_weight"), greaterThan(0L));
@@ -1447,7 +1435,7 @@ public void testCollapseQuerySearchResults() throws Exception {
                 assertThat(query.getTimeBreakdown().get("min_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("avg_score"), greaterThan(0L));
                 assertThat(query.getTimeBreakdown().get("max_score_count"), greaterThanOrEqualTo(6L));
-                assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThanOrEqualTo(6L));
+                assertThat(query.getTimeBreakdown().get("min_score_count"), greaterThanOrEqualTo(2L));
                 assertThat(query.getTimeBreakdown().get("avg_score_count"), greaterThanOrEqualTo(6L));
             }
             assertThat(query.getTimeBreakdown().get("create_weight"), greaterThan(0L));

From 489de2a49a51e6a9a82fbfeefe29bc587b36206b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 11 Oct 2023 11:38:15 -0400
Subject: [PATCH 148/388] Bump commons-io:commons-io from 2.13.0 to 2.14.0 in
 /plugins/ingest-attachment (#10294)

* Bump commons-io:commons-io in /plugins/ingest-attachment

Bumps commons-io:commons-io from 2.13.0 to 2.14.0.

---
updated-dependencies:
- dependency-name: commons-io:commons-io
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 

* Updating SHAs

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 CHANGELOG.md                                                  | 1 +
 plugins/ingest-attachment/build.gradle                        | 2 +-
 plugins/ingest-attachment/licenses/commons-io-2.13.0.jar.sha1 | 1 -
 plugins/ingest-attachment/licenses/commons-io-2.14.0.jar.sha1 | 1 +
 4 files changed, 3 insertions(+), 2 deletions(-)
 delete mode 100644 plugins/ingest-attachment/licenses/commons-io-2.13.0.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/commons-io-2.14.0.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2e107d2c04539..563f3e43e9707 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -123,6 +123,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
 - Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
+- Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
 
 ### Changed
 - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index 330a17c02bc7a..8945c09fca28b 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -57,7 +57,7 @@ dependencies {
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   // Other dependencies
   api 'org.tukaani:xz:1.9'
-  api 'commons-io:commons-io:2.13.0'
+  api 'commons-io:commons-io:2.14.0'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 
   // character set detection
diff --git a/plugins/ingest-attachment/licenses/commons-io-2.13.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-io-2.13.0.jar.sha1
deleted file mode 100644
index c165136eb5822..0000000000000
--- a/plugins/ingest-attachment/licenses/commons-io-2.13.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8bb2bc9b4df17e2411533a0708a69f983bf5e83b
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-io-2.14.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-io-2.14.0.jar.sha1
new file mode 100644
index 0000000000000..33c5cfe53e01d
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-io-2.14.0.jar.sha1
@@ -0,0 +1 @@
+a4c6e1f6c196339473cd2e1b037f0eb97c62755b
\ No newline at end of file

From c55e1b47bffd40defc86686e9c6961d32dd087c2 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Wed, 11 Oct 2023 14:03:49 -0400
Subject: [PATCH 149/388] Add the means to extract the contextual properties
 from HttpChannel, TcpChannel and TrasportChannel without excessive
 typecasting (#10562)

* Add the means to extract the contextual properties from HttpChannel, TcpChannel and TrasportChannel without excessive typecasting

Signed-off-by: Andriy Redko 

* Address code review comments

Signed-off-by: Andriy Redko 

---------

Signed-off-by: Andriy Redko 
---
 CHANGELOG.md                                    |  1 +
 .../http/netty4/Netty4HttpChannel.java          | 17 +++++++++++++++++
 .../transport/netty4/Netty4TcpChannel.java      | 13 +++++++++++++
 .../java/org/opensearch/http/HttpChannel.java   | 14 ++++++++++++++
 .../tracing/channels/TraceableHttpChannel.java  |  6 ++++++
 .../channels/TraceableTcpTransportChannel.java  |  6 ++++++
 .../transport/TaskTransportChannel.java         |  6 ++++++
 .../org/opensearch/transport/TcpChannel.java    | 15 +++++++++++++++
 .../transport/TcpTransportChannel.java          |  5 +++++
 .../opensearch/transport/TransportChannel.java  | 15 +++++++++++++++
 10 files changed, 98 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 563f3e43e9707..5e31085eb3935 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -139,6 +139,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569))
 - [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241))
 - Updating the separator for RemoteStoreLockManager since underscore is allowed in base64UUID url charset ([#10379](https://github.com/opensearch-project/OpenSearch/pull/10379))
+- Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
 
 ### Deprecated
 
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpChannel.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpChannel.java
index a83330356e35e..6475a0b744c60 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpChannel.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpChannel.java
@@ -40,6 +40,7 @@
 import org.opensearch.transport.netty4.Netty4TcpChannel;
 
 import java.net.InetSocketAddress;
+import java.util.Optional;
 
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelPipeline;
@@ -98,6 +99,22 @@ public Channel getNettyChannel() {
         return channel;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public  Optional get(String name, Class clazz) {
+        Object handler = getNettyChannel().pipeline().get(name);
+
+        if (handler == null && inboundPipeline() != null) {
+            handler = inboundPipeline().get(name);
+        }
+
+        if (handler != null && clazz.isInstance(handler) == true) {
+            return Optional.of((T) handler);
+        }
+
+        return Optional.empty();
+    }
+
     @Override
     public String toString() {
         return "Netty4HttpChannel{" + "localAddress=" + getLocalAddress() + ", remoteAddress=" + getRemoteAddress() + '}';
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java
index 5db1f7c333157..79a5bf9e95121 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4TcpChannel.java
@@ -41,6 +41,7 @@
 import org.opensearch.transport.TransportException;
 
 import java.net.InetSocketAddress;
+import java.util.Optional;
 
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFuture;
@@ -164,6 +165,18 @@ public void sendMessage(BytesReference reference, ActionListener listener)
         }
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public  Optional get(String name, Class clazz) {
+        final Object handler = getNettyChannel().pipeline().get(name);
+
+        if (handler != null && clazz.isInstance(handler) == true) {
+            return Optional.of((T) handler);
+        }
+
+        return Optional.empty();
+    }
+
     public Channel getNettyChannel() {
         return channel;
     }
diff --git a/server/src/main/java/org/opensearch/http/HttpChannel.java b/server/src/main/java/org/opensearch/http/HttpChannel.java
index 6dcdaf9034413..679a5d73c7837 100644
--- a/server/src/main/java/org/opensearch/http/HttpChannel.java
+++ b/server/src/main/java/org/opensearch/http/HttpChannel.java
@@ -36,6 +36,7 @@
 import org.opensearch.core.action.ActionListener;
 
 import java.net.InetSocketAddress;
+import java.util.Optional;
 
 /**
  * Represents an HTTP comms channel
@@ -72,4 +73,17 @@ default void handleException(Exception ex) {}
      */
     InetSocketAddress getRemoteAddress();
 
+    /**
+     * Returns the contextual property associated with this specific HTTP channel (the
+     * implementation of how such properties are managed depends on the the particular
+     * transport engine).
+     *
+     * @param name the name of the property
+     * @param clazz the expected type of the property
+     *
+     * @return the value of the property
+     */
+    default  Optional get(String name, Class clazz) {
+        return Optional.empty();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java
index 03848e8e58207..0a9757310fe8b 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java
@@ -18,6 +18,7 @@
 
 import java.net.InetSocketAddress;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Tracer wrapped {@link HttpChannel}
@@ -92,4 +93,9 @@ public InetSocketAddress getLocalAddress() {
     public InetSocketAddress getRemoteAddress() {
         return delegate.getRemoteAddress();
     }
+
+    @Override
+    public  Optional get(String name, Class clazz) {
+        return delegate.get(name, clazz);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java
index 333e06eb037cb..bd60c35c3baac 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java
@@ -20,6 +20,7 @@
 import org.opensearch.transport.TransportChannel;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * Tracer wrapped {@link TransportChannel}
@@ -109,4 +110,9 @@ public void sendResponse(Exception exception) throws IOException {
     public Version getVersion() {
         return delegate.getVersion();
     }
+
+    @Override
+    public  Optional get(String name, Class clazz) {
+        return delegate.get(name, clazz);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/transport/TaskTransportChannel.java b/server/src/main/java/org/opensearch/transport/TaskTransportChannel.java
index 052611317f174..4dab0039ec878 100644
--- a/server/src/main/java/org/opensearch/transport/TaskTransportChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TaskTransportChannel.java
@@ -37,6 +37,7 @@
 import org.opensearch.core.transport.TransportResponse;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * Transport channel for tasks
@@ -89,4 +90,9 @@ public Version getVersion() {
     public TransportChannel getChannel() {
         return channel;
     }
+
+    @Override
+    public  Optional get(String name, Class clazz) {
+        return getChannel().get(name, clazz);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/transport/TcpChannel.java b/server/src/main/java/org/opensearch/transport/TcpChannel.java
index eac137ec30f1a..f98b65d0a4df1 100644
--- a/server/src/main/java/org/opensearch/transport/TcpChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TcpChannel.java
@@ -38,6 +38,7 @@
 import org.opensearch.core.common.bytes.BytesReference;
 
 import java.net.InetSocketAddress;
+import java.util.Optional;
 
 /**
  * This is a tcp channel representing a single channel connection to another node. It is the base channel
@@ -96,6 +97,20 @@ public interface TcpChannel extends CloseableChannel {
      */
     ChannelStats getChannelStats();
 
+    /**
+     * Returns the contextual property associated with this specific TCP channel (the
+     * implementation of how such properties are managed depends on the the particular
+     * transport engine).
+     *
+     * @param name the name of the property
+     * @param clazz the expected type of the property
+     *
+     * @return the value of the property
+     */
+    default  Optional get(String name, Class clazz) {
+        return Optional.empty();
+    }
+
     /**
      * Channel statistics
      *
diff --git a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
index a7bedcf93e129..81de0af07ea7c 100644
--- a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
@@ -38,6 +38,7 @@
 import org.opensearch.search.query.QuerySearchResult;
 
 import java.io.IOException;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -130,4 +131,8 @@ public Version getVersion() {
         return version;
     }
 
+    @Override
+    public  Optional get(String name, Class clazz) {
+        return getChannel().get(name, clazz);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/transport/TransportChannel.java b/server/src/main/java/org/opensearch/transport/TransportChannel.java
index 3c582127f28e8..7423d59103302 100644
--- a/server/src/main/java/org/opensearch/transport/TransportChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TransportChannel.java
@@ -39,6 +39,7 @@
 import org.opensearch.core.transport.TransportResponse;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * A transport channel allows to send a response to a request on the channel.
@@ -78,4 +79,18 @@ static void sendErrorResponse(TransportChannel channel, String actionName, Trans
             );
         }
     }
+
+    /**
+     * Returns the contextual property associated with this specific transport channel (the
+     * implementation of how such properties are managed depends on the the particular
+     * transport engine).
+     *
+     * @param name the name of the property
+     * @param clazz the expected type of the property
+     *
+     * @return the value of the property.
+     */
+    default  Optional get(String name, Class clazz) {
+        return Optional.empty();
+    }
 }

From a54108c6ef378dbbe432e6248240d4f1afd74047 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Wed, 11 Oct 2023 14:04:18 -0400
Subject: [PATCH 150/388] Bump netty from 4.1.99.Final to 4.1.100.Final
 (#10564)

Signed-off-by: Andriy Redko 
---
 CHANGELOG.md                                                    | 1 +
 buildSrc/version.properties                                     | 2 +-
 .../licenses/netty-buffer-4.1.100.Final.jar.sha1                | 1 +
 .../licenses/netty-buffer-4.1.99.Final.jar.sha1                 | 1 -
 .../licenses/netty-codec-4.1.100.Final.jar.sha1                 | 1 +
 .../transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1 | 1 -
 .../licenses/netty-codec-http-4.1.100.Final.jar.sha1            | 1 +
 .../licenses/netty-codec-http-4.1.99.Final.jar.sha1             | 1 -
 .../licenses/netty-codec-http2-4.1.100.Final.jar.sha1           | 1 +
 .../licenses/netty-codec-http2-4.1.99.Final.jar.sha1            | 1 -
 .../licenses/netty-common-4.1.100.Final.jar.sha1                | 1 +
 .../licenses/netty-common-4.1.99.Final.jar.sha1                 | 1 -
 .../licenses/netty-handler-4.1.100.Final.jar.sha1               | 1 +
 .../licenses/netty-handler-4.1.99.Final.jar.sha1                | 1 -
 .../licenses/netty-resolver-4.1.100.Final.jar.sha1              | 1 +
 .../licenses/netty-resolver-4.1.99.Final.jar.sha1               | 1 -
 .../licenses/netty-transport-4.1.100.Final.jar.sha1             | 1 +
 .../licenses/netty-transport-4.1.99.Final.jar.sha1              | 1 -
 .../netty-transport-native-unix-common-4.1.100.Final.jar.sha1   | 1 +
 .../netty-transport-native-unix-common-4.1.99.Final.jar.sha1    | 1 -
 .../licenses/netty-codec-dns-4.1.100.Final.jar.sha1             | 1 +
 .../licenses/netty-codec-dns-4.1.99.Final.jar.sha1              | 1 -
 .../licenses/netty-codec-http2-4.1.100.Final.jar.sha1           | 1 +
 .../licenses/netty-codec-http2-4.1.99.Final.jar.sha1            | 1 -
 .../licenses/netty-codec-socks-4.1.100.Final.jar.sha1           | 1 +
 .../licenses/netty-codec-socks-4.1.99.Final.jar.sha1            | 1 -
 .../licenses/netty-handler-proxy-4.1.100.Final.jar.sha1         | 1 +
 .../licenses/netty-handler-proxy-4.1.99.Final.jar.sha1          | 1 -
 .../licenses/netty-resolver-dns-4.1.100.Final.jar.sha1          | 1 +
 .../licenses/netty-resolver-dns-4.1.99.Final.jar.sha1           | 1 -
 .../netty-transport-native-unix-common-4.1.100.Final.jar.sha1   | 1 +
 .../netty-transport-native-unix-common-4.1.99.Final.jar.sha1    | 1 -
 .../repository-hdfs/licenses/netty-all-4.1.100.Final.jar.sha1   | 1 +
 .../repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1    | 1 -
 .../repository-s3/licenses/netty-buffer-4.1.100.Final.jar.sha1  | 1 +
 .../repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1   | 1 -
 .../repository-s3/licenses/netty-codec-4.1.100.Final.jar.sha1   | 1 +
 .../repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1    | 1 -
 .../licenses/netty-codec-http-4.1.100.Final.jar.sha1            | 1 +
 .../licenses/netty-codec-http-4.1.99.Final.jar.sha1             | 1 -
 .../licenses/netty-codec-http2-4.1.100.Final.jar.sha1           | 1 +
 .../licenses/netty-codec-http2-4.1.99.Final.jar.sha1            | 1 -
 .../repository-s3/licenses/netty-common-4.1.100.Final.jar.sha1  | 1 +
 .../repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1   | 1 -
 .../repository-s3/licenses/netty-handler-4.1.100.Final.jar.sha1 | 1 +
 .../repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1  | 1 -
 .../licenses/netty-resolver-4.1.100.Final.jar.sha1              | 1 +
 .../repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1 | 1 -
 .../licenses/netty-transport-4.1.100.Final.jar.sha1             | 1 +
 .../licenses/netty-transport-4.1.99.Final.jar.sha1              | 1 -
 .../netty-transport-classes-epoll-4.1.100.Final.jar.sha1        | 1 +
 .../netty-transport-classes-epoll-4.1.99.Final.jar.sha1         | 1 -
 .../netty-transport-native-unix-common-4.1.100.Final.jar.sha1   | 1 +
 .../netty-transport-native-unix-common-4.1.99.Final.jar.sha1    | 1 -
 .../transport-nio/licenses/netty-buffer-4.1.100.Final.jar.sha1  | 1 +
 .../transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1   | 1 -
 .../transport-nio/licenses/netty-codec-4.1.100.Final.jar.sha1   | 1 +
 .../transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1    | 1 -
 .../licenses/netty-codec-http-4.1.100.Final.jar.sha1            | 1 +
 .../licenses/netty-codec-http-4.1.99.Final.jar.sha1             | 1 -
 .../transport-nio/licenses/netty-common-4.1.100.Final.jar.sha1  | 1 +
 .../transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1   | 1 -
 .../transport-nio/licenses/netty-handler-4.1.100.Final.jar.sha1 | 1 +
 .../transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1  | 1 -
 .../licenses/netty-resolver-4.1.100.Final.jar.sha1              | 1 +
 .../transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1 | 1 -
 .../licenses/netty-transport-4.1.100.Final.jar.sha1             | 1 +
 .../licenses/netty-transport-4.1.99.Final.jar.sha1              | 1 -
 68 files changed, 35 insertions(+), 34 deletions(-)
 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-common-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.100.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5e31085eb3935..6055af47403bc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963))
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
+- Bump `netty` from 4.1.99.Final to 4.1.100.Final ([#10564](https://github.com/opensearch-project/OpenSearch/pull/10564))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index e54a5a1089a93..64f9f9a8828ad 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -28,7 +28,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.99.Final
+netty             = 4.1.100.Final
 joda              = 2.12.2
 
 # client dependencies
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..aaf2e35302d77
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+39b05d2d4027971bf99111a9be1d7035a116bb55
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1
deleted file mode 100644
index 5b393be40e945..0000000000000
--- a/modules/transport-netty4/licenses/netty-buffer-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9f02dcb9b15a647a56af210dffdc294a57922fb0
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..a77333ea8ae47
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+9c3c71e7cf3b8ce3bfc9fa52a524b9ca7ddf259c
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1
deleted file mode 100644
index 45ea27d29a183..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9984cbd6e5d55c768f198e975d8aaf7fd42a4602
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..6f26bf4e6a9b5
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+992623e7d8f2d96e41faf1687bb963f5433e3517
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1
deleted file mode 100644
index 6bb7fcd68b272..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7142095066eaebd5f29b88c41af7b383b6a953f6
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..bf5605151406e
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+cbf1a430ea44dbdedbcde16b185cbb95f28d72c7
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
deleted file mode 100644
index f9bdefc6dd965..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c5a3481c4bb9732a3a94fb63cf916141a1a14669
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..d2ff72db60d1f
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+847f942381145de23f21c836d05b0677474271d3
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1
deleted file mode 100644
index d53adfa649f5f..0000000000000
--- a/modules/transport-netty4/licenses/netty-common-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-278f6dfa49d6bd75c40ae1470eb165716f87dce0
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..f12a6046e96d0
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+4c0acdb8bb73647ebb3847ac2d503d53d72c02b4
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1
deleted file mode 100644
index 258f7c957dda0..0000000000000
--- a/modules/transport-netty4/licenses/netty-handler-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-742693761d7ea4c038bccfda96bb38194720b80d
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..8e4179ba15942
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+fe62f9ccd41b8660d07639dbbab8ae1edd6f2720
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1
deleted file mode 100644
index b8bc0a4370f58..0000000000000
--- a/modules/transport-netty4/licenses/netty-resolver-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-080e45397d9d5b134477de3ffd0f94283b908621
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..ab2819da570fd
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+6620fbfb47667a5eb6050e35c7b4c88000bcd77f
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1
deleted file mode 100644
index 247975e0a64c7..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ca2e3ae19a6713b749df154622115f480b6716c
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..30d7758302e37
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+648ff5571022dbfa6789122e3872477bbf67fa7b
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
deleted file mode 100644
index 6b7b66ea768e3..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cb0fc6c31c387404212949c57950b5d72ce908b9
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.100.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..dfa4a0fbea94c
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+663b1b7bf3ff0f12fde4df20c72d9e94584ebffa
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1
deleted file mode 100644
index 6c1112ed49775..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-21c76a42a468faafac6c84f8aca775073fc8e345
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.100.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..bf5605151406e
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+cbf1a430ea44dbdedbcde16b185cbb95f28d72c7
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
deleted file mode 100644
index f9bdefc6dd965..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c5a3481c4bb9732a3a94fb63cf916141a1a14669
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.100.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..8e9bc8c96aec7
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+a9fbf4d64b08abed542eefd5f7aed4807edca56f
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1
deleted file mode 100644
index 717703c36e1ab..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-259bf1c5178c3e23bb89a2fab59b6d22846e3fa6
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.100.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..35d9d82202274
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+af3cf676eed30184215426ecf0f0dde15555ea9c
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1
deleted file mode 100644
index 3f69ae54c5d4a..0000000000000
--- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8c8a89ea89b06e120c57bdb3db14b9a47ca30bb3
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..0948daa05fff6
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+62dbdf5f25eda75ea8456be1ed72b3fcb0d18774
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1
deleted file mode 100644
index adef44a4e7da7..0000000000000
--- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-804d8b752847923d3bb81f24de604597047c9b2e
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..30d7758302e37
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+648ff5571022dbfa6789122e3872477bbf67fa7b
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
deleted file mode 100644
index 6b7b66ea768e3..0000000000000
--- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cb0fc6c31c387404212949c57950b5d72ce908b9
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.100.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..a9aa34392903e
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/netty-all-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+5ef15a3ce29a792b7ad17438e5f84c617b3f2993
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1
deleted file mode 100644
index 0756635018837..0000000000000
--- a/plugins/repository-hdfs/licenses/netty-all-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a45aa70bc50d0500da5cdcd595cc838d87ada987
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..aaf2e35302d77
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-buffer-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+39b05d2d4027971bf99111a9be1d7035a116bb55
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1
deleted file mode 100644
index 5b393be40e945..0000000000000
--- a/plugins/repository-s3/licenses/netty-buffer-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9f02dcb9b15a647a56af210dffdc294a57922fb0
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..a77333ea8ae47
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+9c3c71e7cf3b8ce3bfc9fa52a524b9ca7ddf259c
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1
deleted file mode 100644
index 45ea27d29a183..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9984cbd6e5d55c768f198e975d8aaf7fd42a4602
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..6f26bf4e6a9b5
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+992623e7d8f2d96e41faf1687bb963f5433e3517
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1
deleted file mode 100644
index 6bb7fcd68b272..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-http-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7142095066eaebd5f29b88c41af7b383b6a953f6
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..bf5605151406e
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+cbf1a430ea44dbdedbcde16b185cbb95f28d72c7
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
deleted file mode 100644
index f9bdefc6dd965..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c5a3481c4bb9732a3a94fb63cf916141a1a14669
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-common-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..d2ff72db60d1f
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+847f942381145de23f21c836d05b0677474271d3
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1
deleted file mode 100644
index d53adfa649f5f..0000000000000
--- a/plugins/repository-s3/licenses/netty-common-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-278f6dfa49d6bd75c40ae1470eb165716f87dce0
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..f12a6046e96d0
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-handler-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+4c0acdb8bb73647ebb3847ac2d503d53d72c02b4
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1
deleted file mode 100644
index 258f7c957dda0..0000000000000
--- a/plugins/repository-s3/licenses/netty-handler-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-742693761d7ea4c038bccfda96bb38194720b80d
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..8e4179ba15942
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-resolver-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+fe62f9ccd41b8660d07639dbbab8ae1edd6f2720
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1
deleted file mode 100644
index b8bc0a4370f58..0000000000000
--- a/plugins/repository-s3/licenses/netty-resolver-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-080e45397d9d5b134477de3ffd0f94283b908621
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..ab2819da570fd
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+6620fbfb47667a5eb6050e35c7b4c88000bcd77f
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1
deleted file mode 100644
index 247975e0a64c7..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ca2e3ae19a6713b749df154622115f480b6716c
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..5805fdaf411d1
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+78489936ca1d91483e34a31d04a3b0812386eb39
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1
deleted file mode 100644
index 75b64ad4197d8..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-961bd5b8d97ea6a07168176462f398089a24b5c8
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..30d7758302e37
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+648ff5571022dbfa6789122e3872477bbf67fa7b
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
deleted file mode 100644
index 6b7b66ea768e3..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cb0fc6c31c387404212949c57950b5d72ce908b9
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.100.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..aaf2e35302d77
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-buffer-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+39b05d2d4027971bf99111a9be1d7035a116bb55
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1
deleted file mode 100644
index 5b393be40e945..0000000000000
--- a/plugins/transport-nio/licenses/netty-buffer-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9f02dcb9b15a647a56af210dffdc294a57922fb0
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.100.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..a77333ea8ae47
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+9c3c71e7cf3b8ce3bfc9fa52a524b9ca7ddf259c
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1
deleted file mode 100644
index 45ea27d29a183..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9984cbd6e5d55c768f198e975d8aaf7fd42a4602
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.100.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..6f26bf4e6a9b5
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+992623e7d8f2d96e41faf1687bb963f5433e3517
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1
deleted file mode 100644
index 6bb7fcd68b272..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-http-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7142095066eaebd5f29b88c41af7b383b6a953f6
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.100.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..d2ff72db60d1f
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+847f942381145de23f21c836d05b0677474271d3
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1
deleted file mode 100644
index d53adfa649f5f..0000000000000
--- a/plugins/transport-nio/licenses/netty-common-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-278f6dfa49d6bd75c40ae1470eb165716f87dce0
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.100.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..f12a6046e96d0
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-handler-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+4c0acdb8bb73647ebb3847ac2d503d53d72c02b4
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1
deleted file mode 100644
index 258f7c957dda0..0000000000000
--- a/plugins/transport-nio/licenses/netty-handler-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-742693761d7ea4c038bccfda96bb38194720b80d
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.100.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..8e4179ba15942
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-resolver-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+fe62f9ccd41b8660d07639dbbab8ae1edd6f2720
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1
deleted file mode 100644
index b8bc0a4370f58..0000000000000
--- a/plugins/transport-nio/licenses/netty-resolver-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-080e45397d9d5b134477de3ffd0f94283b908621
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.100.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..ab2819da570fd
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-transport-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+6620fbfb47667a5eb6050e35c7b4c88000bcd77f
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1
deleted file mode 100644
index 247975e0a64c7..0000000000000
--- a/plugins/transport-nio/licenses/netty-transport-4.1.99.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ca2e3ae19a6713b749df154622115f480b6716c
\ No newline at end of file

From f518e9dfd518e1ad31d62ff028b61ab91222204d Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Wed, 11 Oct 2023 21:17:10 -0400
Subject: [PATCH 151/388] Update OpenSearch 2.12 to use Apache Lucene 9.8.0
 (#10578)

Signed-off-by: Andriy Redko 
---
 libs/core/src/main/java/org/opensearch/Version.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index 3c210e5c7cf96..eef4da719994c 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -96,7 +96,7 @@ public class Version implements Comparable, ToXContentFragment {
     public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_10_1 = new Version(2100199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0);
-    public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_7_0);
+    public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0);
     public static final Version CURRENT = V_3_0_0;
 

From 90c4297266495de41c3b09728ce82d1a9aec1d64 Mon Sep 17 00:00:00 2001
From: Ashish 
Date: Thu, 12 Oct 2023 10:26:34 +0530
Subject: [PATCH 152/388] Fix shard failure on flush during upload failures for
 remote indexes (#10513)

Signed-off-by: Ashish Singh 
---
 ...emoteStoreBackpressureAndResiliencyIT.java | 29 +++++++++++++++++++
 .../opensearch/index/shard/IndexShard.java    |  2 +-
 .../translog/InternalTranslogManager.java     |  6 +++-
 .../index/translog/TranslogManager.java       |  2 +-
 .../transfer/TranslogTransferManager.java     |  4 +--
 .../TranslogUploadFailedException.java        | 27 +++++++++++++++++
 .../index/engine/InternalEngineTests.java     |  6 +++-
 .../index/translog/RemoteFsTranslogTests.java |  3 +-
 8 files changed, 72 insertions(+), 7 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/translog/transfer/TranslogUploadFailedException.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
index 2c6db6ae19a9a..98586b60dcc69 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
@@ -11,6 +11,7 @@
 import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats;
 import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.opensearch.action.admin.indices.flush.FlushResponse;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.AbstractAsyncTask;
@@ -228,4 +229,32 @@ public void testSkipLoadGlobalCheckpointToReplicationTracker() {
         client().admin().cluster().prepareReroute().setRetryFailed(true).get();
         ensureGreen(INDEX_NAME);
     }
+
+    public void testFlushDuringRemoteUploadFailures() {
+        Path location = randomRepoPath().toAbsolutePath();
+        String dataNodeName = setup(location, 0d, "metadata", Long.MAX_VALUE);
+
+        logger.info("--> Indexing data");
+        indexData(randomIntBetween(1, 2), true);
+        logger.info("--> Indexing succeeded");
+        ensureGreen(INDEX_NAME);
+
+        MockRepository translogRepo = (MockRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName)
+            .repository(TRANSLOG_REPOSITORY_NAME);
+        logger.info("--> Failing all remote store interaction");
+        translogRepo.setRandomControlIOExceptionRate(1d);
+
+        Exception ex = assertThrows(UncategorizedExecutionException.class, () -> indexSingleDoc());
+        assertEquals("Failed execution", ex.getMessage());
+
+        FlushResponse flushResponse = client().admin().indices().prepareFlush(INDEX_NAME).setForce(true).execute().actionGet();
+        assertEquals(1, flushResponse.getFailedShards());
+        ensureGreen(INDEX_NAME);
+
+        logger.info("--> Stop failing all remote store interactions");
+        translogRepo.setRandomControlIOExceptionRate(0d);
+        flushResponse = client().admin().indices().prepareFlush(INDEX_NAME).setForce(true).execute().actionGet();
+        assertEquals(1, flushResponse.getSuccessfulShards());
+        assertEquals(0, flushResponse.getFailedShards());
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 6ac75c7bfec17..9489c7d7fc1dd 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -1475,7 +1475,7 @@ public void trimTranslog() {
     /**
      * Rolls the tranlog generation and cleans unneeded.
      */
-    public void rollTranslogGeneration() {
+    public void rollTranslogGeneration() throws IOException {
         final Engine engine = getEngine();
         engine.translogManager().rollTranslogGeneration();
     }
diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
index 4bc9a711894b7..85c52b907d326 100644
--- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
@@ -18,6 +18,7 @@
 import org.opensearch.index.engine.LifecycleAware;
 import org.opensearch.index.seqno.LocalCheckpointTracker;
 import org.opensearch.index.translog.listener.TranslogEventListener;
+import org.opensearch.index.translog.transfer.TranslogUploadFailedException;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -83,11 +84,14 @@ public InternalTranslogManager(
      * Rolls the translog generation and cleans unneeded.
      */
     @Override
-    public void rollTranslogGeneration() throws TranslogException {
+    public void rollTranslogGeneration() throws TranslogException, IOException {
         try (ReleasableLock ignored = readLock.acquire()) {
             engineLifeCycleAware.ensureOpen();
             translog.rollGeneration();
             translog.trimUnreferencedReaders();
+        } catch (TranslogUploadFailedException e) {
+            // Do not trigger the translogEventListener as it fails the Engine while this is only an issue with remote upload
+            throw e;
         } catch (AlreadyClosedException e) {
             translogEventListener.onFailure("translog roll generation failed", e);
             throw e;
diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogManager.java b/server/src/main/java/org/opensearch/index/translog/TranslogManager.java
index 78aaa1bc13a00..148fd67fb413e 100644
--- a/server/src/main/java/org/opensearch/index/translog/TranslogManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/TranslogManager.java
@@ -21,7 +21,7 @@ public interface TranslogManager {
     /**
      * Rolls the translog generation and cleans unneeded.
      */
-    void rollTranslogGeneration() throws TranslogException;
+    void rollTranslogGeneration() throws TranslogException, IOException;
 
     /**
      * Performs recovery from the transaction log up to {@code recoverUpToSeqNo} (inclusive).
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
index d988b8a6254ff..ece6f6d5a534f 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
@@ -176,7 +176,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
                     remoteTranslogTransferTracker.addUploadTimeInMillis((System.nanoTime() - metadataUploadStartTime) / 1_000_000L);
                     remoteTranslogTransferTracker.addUploadBytesFailed(metadataBytesToUpload);
                     // outer catch handles capturing stats on upload failure
-                    throw exception;
+                    throw new TranslogUploadFailedException("Failed to upload " + tlogMetadata.getName(), exception);
                 }
 
                 remoteTranslogTransferTracker.addUploadTimeInMillis((System.nanoTime() - metadataUploadStartTime) / 1_000_000L);
@@ -185,7 +185,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
                 translogTransferListener.onUploadComplete(transferSnapshot);
                 return true;
             } else {
-                Exception ex = new IOException("Failed to upload " + exceptionList.size() + " files during transfer");
+                Exception ex = new TranslogUploadFailedException("Failed to upload " + exceptionList.size() + " files during transfer");
                 exceptionList.forEach(ex::addSuppressed);
                 throw ex;
             }
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogUploadFailedException.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogUploadFailedException.java
new file mode 100644
index 0000000000000..4a9b10ec5a52e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogUploadFailedException.java
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.translog.transfer;
+
+import java.io.IOException;
+
+/**
+ * Exception is thrown if there are any exceptions while uploading translog to remote store.
+ * @opensearch.internal
+ */
+public class TranslogUploadFailedException extends IOException {
+
+    public TranslogUploadFailedException(String message) {
+        super(message);
+    }
+
+    public TranslogUploadFailedException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
index b8bb73bb89a82..305c3a3acbf75 100644
--- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
@@ -7256,7 +7256,11 @@ public void testMaxSeqNoInCommitUserData() throws Exception {
             engine.ensureOpen();
             while (running.get()
                 && assertAndGetInternalTranslogManager(engine.translogManager()).getTranslog().currentFileGeneration() < 500) {
-                engine.translogManager().rollTranslogGeneration(); // make adding operations to translog slower
+                try {
+                    engine.translogManager().rollTranslogGeneration(); // make adding operations to translog slower
+                } catch (IOException e) {
+                    fail("io exception not expected");
+                }
             }
         });
         rollTranslog.start();
diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
index b2310010620f7..42e0df2dc90c1 100644
--- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
@@ -47,6 +47,7 @@
 import org.opensearch.index.translog.transfer.BlobStoreTransferService;
 import org.opensearch.index.translog.transfer.TranslogTransferManager;
 import org.opensearch.index.translog.transfer.TranslogTransferMetadata;
+import org.opensearch.index.translog.transfer.TranslogUploadFailedException;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
@@ -1113,7 +1114,7 @@ public void testSyncUpAlwaysFailure() throws IOException {
             try {
                 translog.sync();
                 fail("io exception expected");
-            } catch (IOException e) {
+            } catch (TranslogUploadFailedException e) {
                 assertTrue("at least one operation pending", translog.syncNeeded());
             }
         }

From 9c06228f2e39b3a7b46e9eb26aa2368dcbf4116a Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Thu, 12 Oct 2023 16:09:11 +0530
Subject: [PATCH 153/388] Fix system_repository setting is restored after
 cluster metadata recovery from local disk (#10556)

Signed-off-by: bansvaru 
---
 .../RemoteStoreRepositoryRegistrationIT.java  | 19 +++++++++++++++++++
 .../get/GetRepositoriesResponse.java          |  7 +++++--
 .../metadata/RepositoriesMetadata.java        |  3 ++-
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java
index 002a149f0c286..ef2dcf3217df6 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java
@@ -8,21 +8,31 @@
 
 package org.opensearch.remotestore;
 
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesAction;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.disruption.NetworkDisruption;
 import org.opensearch.test.transport.MockTransportService;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING;
+
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreRepositoryRegistrationIT extends RemoteStoreBaseIntegTestCase {
 
@@ -162,4 +172,13 @@ public void testNodeRestartPostNonRestrictedSettingsUpdate() throws Exception {
 
         ensureStableCluster(4);
     }
+
+    public void testSystemRepositorySettingIsHiddenForGetRepositoriesRequest() throws IOException {
+        GetRepositoriesRequest request = new GetRepositoriesRequest(new String[] { REPOSITORY_NAME });
+        GetRepositoriesResponse repositoriesResponse = client().execute(GetRepositoriesAction.INSTANCE, request).actionGet();
+        XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.JSON));
+        XContentBuilder xContentBuilder = repositoriesResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        repositoriesResponse = GetRepositoriesResponse.fromXContent(createParser(xContentBuilder));
+        assertEquals(false, SYSTEM_REPOSITORY_SETTING.get(repositoriesResponse.repositories().get(0).settings()));
+    }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
index 098a0e60142e7..f8c8df25be532 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
@@ -42,8 +42,8 @@
 import org.opensearch.core.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;
 
@@ -83,7 +83,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         repositories.toXContent(
             builder,
-            new DelegatingMapParams(Collections.singletonMap(RepositoriesMetadata.HIDE_GENERATIONS_PARAM, "true"), params)
+            new DelegatingMapParams(
+                Map.of(RepositoriesMetadata.HIDE_GENERATIONS_PARAM, "true", RepositoriesMetadata.HIDE_SYSTEM_REPOSITORY_SETTING, "true"),
+                params
+            )
         );
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java
index a5ef337c3b62a..e3689d046193c 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java
@@ -70,6 +70,7 @@ public class RepositoriesMetadata extends AbstractNamedDiffable implemen
      * in {@link org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse}.
      */
     public static final String HIDE_GENERATIONS_PARAM = "hide_generations";
+    public static final String HIDE_SYSTEM_REPOSITORY_SETTING = "hide_system_repository_setting";
 
     private final List repositories;
 
@@ -291,7 +292,7 @@ public static void toXContent(RepositoryMetadata repository, XContentBuilder bui
             repository.cryptoMetadata().toXContent(repository.cryptoMetadata(), builder, params);
         }
         Settings settings = repository.settings();
-        if (SYSTEM_REPOSITORY_SETTING.get(settings)) {
+        if (SYSTEM_REPOSITORY_SETTING.get(settings) && params.paramAsBoolean(HIDE_SYSTEM_REPOSITORY_SETTING, false)) {
             settings = repository.settings().filter(s -> !s.equals(SYSTEM_REPOSITORY_SETTING.getKey()));
         }
         builder.startObject("settings");

From 6c022612769e6bc7ed18b9ec0888d6cd0dd415cc Mon Sep 17 00:00:00 2001
From: Sachin Kale 
Date: Thu, 12 Oct 2023 16:11:56 +0530
Subject: [PATCH 154/388] Validate checksum of each segment file post download
 from remote store (#10119)

---------

Signed-off-by: Sachin Kale 
Co-authored-by: Sachin Kale 
---
 .../remotestore/RemoteStoreRestoreIT.java     | 33 ++++++++++++-
 .../store/RemoteSegmentStoreDirectory.java    |  4 ++
 .../org/opensearch/index/store/Store.java     | 47 ++++++++++++++++++-
 3 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
index 7626e3dba6424..212f797180077 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
@@ -10,8 +10,11 @@
 
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse;
+import org.opensearch.action.admin.indices.get.GetIndexRequest;
+import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
@@ -19,10 +22,12 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
+import org.opensearch.test.CorruptionUtils;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Locale;
@@ -30,13 +35,14 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThan;
 
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreRestoreIT extends BaseRemoteStoreRestoreIT {
 
     /**
@@ -461,5 +467,30 @@ public void testRateLimitedRemoteDownloads() throws Exception {
         }
     }
 
+    public void testRestoreCorruptSegmentShouldFail() throws IOException, ExecutionException, InterruptedException {
+        prepareCluster(1, 3, INDEX_NAME, 0, 1);
+        indexData(randomIntBetween(3, 4), true, INDEX_NAME);
+
+        GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest()).get();
+        String indexUUID = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
+
+        logger.info("--> Corrupting segment files in remote segment store");
+        Path path = segmentRepoPath.resolve(indexUUID).resolve("0").resolve("segments").resolve("data");
+        try (Stream dataPath = Files.list(path)) {
+            CorruptionUtils.corruptFile(random(), dataPath.toArray(Path[]::new));
+        }
+
+        logger.info("--> Stop primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME)));
+
+        logger.info("--> Close and restore the index");
+        client().admin()
+            .cluster()
+            .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).waitForCompletion(true), PlainActionFuture.newFuture());
+
+        logger.info("--> Check for index status, should be red due to corruption");
+        ensureRed(INDEX_NAME);
+    }
+
     // TODO: Restore flow - index aliases
 }
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
index be1f2341236ab..6b43fed3d8930 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
@@ -290,6 +290,10 @@ public void setWrittenByMajor(int writtenByMajor) {
                 );
             }
         }
+
+        public int getWrittenByMajor() {
+            return writtenByMajor;
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java
index b822742de6e97..d0cd2635ba672 100644
--- a/server/src/main/java/org/opensearch/index/store/Store.java
+++ b/server/src/main/java/org/opensearch/index/store/Store.java
@@ -105,6 +105,7 @@
 import java.io.UncheckedIOException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -120,6 +121,7 @@
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
+import static java.lang.Character.MAX_RADIX;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.opensearch.index.seqno.SequenceNumbers.LOCAL_CHECKPOINT_KEY;
@@ -975,7 +977,11 @@ public void copyFrom(Directory from, String src, String dest, IOContext context)
             boolean success = false;
             long startTime = System.currentTimeMillis();
             try {
-                super.copyFrom(from, src, dest, context);
+                if (from instanceof RemoteSegmentStoreDirectory) {
+                    copyFileAndValidateChecksum(from, src, dest, context, fileSize);
+                } else {
+                    super.copyFrom(from, src, dest, context);
+                }
                 success = true;
                 afterDownload(fileSize, startTime);
             } finally {
@@ -985,6 +991,43 @@ public void copyFrom(Directory from, String src, String dest, IOContext context)
             }
         }
 
+        private void copyFileAndValidateChecksum(Directory from, String src, String dest, IOContext context, long fileSize)
+            throws IOException {
+            RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = ((RemoteSegmentStoreDirectory) from)
+                .getSegmentsUploadedToRemoteStore()
+                .get(dest);
+            boolean success = false;
+            try (IndexInput is = from.openInput(src, context); IndexOutput os = createOutput(dest, context)) {
+                // Here, we don't need the exact version as LuceneVerifyingIndexOutput does not verify version
+                // It is just used to emit logs when the entire metadata object is provided as parameter. Also,
+                // we can't provide null version as StoreFileMetadata has non-null check on writtenBy field.
+                Version luceneMajorVersion = Version.parse(metadata.getWrittenByMajor() + ".0.0");
+                long checksum = Long.parseLong(metadata.getChecksum());
+                StoreFileMetadata storeFileMetadata = new StoreFileMetadata(
+                    dest,
+                    fileSize,
+                    Long.toString(checksum, MAX_RADIX),
+                    luceneMajorVersion
+                );
+                VerifyingIndexOutput verifyingIndexOutput = new LuceneVerifyingIndexOutput(storeFileMetadata, os);
+                verifyingIndexOutput.copyBytes(is, is.length());
+                verifyingIndexOutput.verify();
+                success = true;
+            } catch (ParseException e) {
+                throw new IOException("Exception while reading version info for segment file from remote store: " + dest, e);
+            } finally {
+                if (success == false) {
+                    // If the exception is thrown after file is created, we clean up the file.
+                    // We ignore the exception as the deletion is best-effort basis and can fail if file does not exist.
+                    try {
+                        deleteFile("Quietly deleting", dest);
+                    } catch (Exception e) {
+                        // Ignore
+                    }
+                }
+            }
+        }
+
         /**
          * Updates the amount of bytes attempted for download
          */
@@ -1476,7 +1519,7 @@ public static boolean isAutogenerated(String name) {
      * Produces a string representation of the given digest value.
      */
     public static String digestToString(long digest) {
-        return Long.toString(digest, Character.MAX_RADIX);
+        return Long.toString(digest, MAX_RADIX);
     }
 
     /**

From 382eb301b21bf5c6eda613432b9ddccda1c414b9 Mon Sep 17 00:00:00 2001
From: Louis Chu 
Date: Thu, 12 Oct 2023 17:31:25 -0400
Subject: [PATCH 155/388] Add release notes for 2.11 (#10594)

Signed-off-by: Louis Chu 
---
 CHANGELOG.md                                  | 65 +---------------
 .../opensearch.release-notes-2.11.0.md        | 75 +++++++++++++++++++
 2 files changed, 76 insertions(+), 64 deletions(-)
 create mode 100644 release-notes/opensearch.release-notes-2.11.0.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6055af47403bc..a9e5bb3982708 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,7 +33,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.code.gson:gson` from 2.10 to 2.10.1
 - Bump `com.maxmind.geoip2:geoip2` from 4.0.0 to 4.0.1
 - Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.11 to 0.16.12
-- Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0
 - Bump `org.apache.commons:commons-configuration2` from 2.8.0 to 2.9.0
 - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
@@ -46,7 +45,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963))
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
-- Bump `netty` from 4.1.99.Final to 4.1.100.Final ([#10564](https://github.com/opensearch-project/OpenSearch/pull/10564))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -86,79 +84,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
-- Add coordinator level stats for search latency ([#8386](https://github.com/opensearch-project/OpenSearch/issues/8386))
-- Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681))
-- Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694))
-- [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666))
-- Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))
-- Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189))
-- Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562))
-- Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204))
-- [Remote Store] Add support to restrict creation & deletion if system repository and mutation of immutable settings of system repository ([#9839](https://github.com/opensearch-project/OpenSearch/pull/9839))
-- Improve compressed request handling ([#10261](https://github.com/opensearch-project/OpenSearch/pull/10261))
 
 ### Dependencies
-- Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575))
-- Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968))
-- Bump OpenTelemetry from 1.26.0 to 1.30.1 ([#9950](https://github.com/opensearch-project/OpenSearch/pull/9950))
-- Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973, #9972](https://github.com/opensearch-project/OpenSearch/pull/9973, https://github.com/opensearch-project/OpenSearch/pull/9972))
-- Bump `com.google.cloud:google-cloud-core-http` from 2.21.1 to 2.23.0 ([#9971](https://github.com/opensearch-project/OpenSearch/pull/9971))
-- Bump `mockito` from 5.4.0 to 5.5.0 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022))
-- Bump `bytebuddy` from 1.14.3 to 1.14.7 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022))
-- Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098))
-- Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125))
-- Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752))
-- Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126))
-- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.5 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206), [#10299](https://github.com/opensearch-project/OpenSearch/pull/10299))
-- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
-- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
-- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
-- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
-- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
-- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
-- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
-- Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302))
-- Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303))
-- Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301))
-- Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210))
-- Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))
-- Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
-- Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
-- Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
 
 ### Changed
-- Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
-- Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916))
-- Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
-- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
-- Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042))
-- [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122))
-- Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246))
-- Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200))
-- Add instrumentation in Inbound Handler. ([#100143](https://github.com/opensearch-project/OpenSearch/pull/10143))
-- Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356))
-- [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569))
-- [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241))
-- Updating the separator for RemoteStoreLockManager since underscore is allowed in base64UUID url charset ([#10379](https://github.com/opensearch-project/OpenSearch/pull/10379))
-- Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
 
 ### Deprecated
 
 ### Removed
-- Remove spurious SGID bit on directories ([#9447](https://github.com/opensearch-project/OpenSearch/pull/9447))
 
 ### Fixed
-- Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725))
-- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045))
-- Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082))
-- Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089))
-- Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
-- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194))
-- Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256))
-- Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
-- Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
 
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
diff --git a/release-notes/opensearch.release-notes-2.11.0.md b/release-notes/opensearch.release-notes-2.11.0.md
new file mode 100644
index 0000000000000..d7e9182f2a656
--- /dev/null
+++ b/release-notes/opensearch.release-notes-2.11.0.md
@@ -0,0 +1,75 @@
+## 2023-10-12 Version 2.11.0 Release Notes
+
+## [2.11]
+
+### Added
+- Add coordinator level stats for search latency ([#8386](https://github.com/opensearch-project/OpenSearch/issues/8386))
+- Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681))
+- Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694))
+- [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666))
+- Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))
+- Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189))
+- Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562))
+- Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204))
+- [Remote Store] Add support to restrict creation & deletion if system repository and mutation of immutable settings of system repository ([#9839](https://github.com/opensearch-project/OpenSearch/pull/9839))
+- Improve compressed request handling ([#10261](https://github.com/opensearch-project/OpenSearch/pull/10261))
+
+### Dependencies
+- Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575))
+- Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968))
+- Bump OpenTelemetry from 1.26.0 to 1.30.1 ([#9950](https://github.com/opensearch-project/OpenSearch/pull/9950))
+- Bump `org.apache.commons:commons-compress` from 1.23.0 to 1.24.0 ([#9973, #9972](https://github.com/opensearch-project/OpenSearch/pull/9973, https://github.com/opensearch-project/OpenSearch/pull/9972))
+- Bump `com.google.cloud:google-cloud-core-http` from 2.21.1 to 2.23.0 ([#9971](https://github.com/opensearch-project/OpenSearch/pull/9971))
+- Bump `mockito` from 5.4.0 to 5.5.0 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022))
+- Bump `bytebuddy` from 1.14.3 to 1.14.7 ([#10022](https://github.com/opensearch-project/OpenSearch/pull/10022))
+- Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098))
+- Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125))
+- Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752))
+- Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126))
+- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.5 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206), [#10299](https://github.com/opensearch-project/OpenSearch/pull/10299))
+- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
+- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
+- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
+- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
+- Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302))
+- Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303))
+- Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301))
+- Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210))
+- Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))
+- Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
+- Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
+- Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
+- Bump `netty` from 4.1.99.Final to 4.1.100.Final ([#10564](https://github.com/opensearch-project/OpenSearch/pull/10564))
+
+### Changed
+- Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
+- Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916))
+- Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
+- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
+- Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042))
+- [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122))
+- Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246))
+- Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200))
+- Add instrumentation in Inbound Handler. ([#100143](https://github.com/opensearch-project/OpenSearch/pull/10143))
+- Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356))
+- [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569))
+- [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241))
+- Updating the separator for RemoteStoreLockManager since underscore is allowed in base64UUID url charset ([#10379](https://github.com/opensearch-project/OpenSearch/pull/10379))
+- Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
+
+### Removed
+- Remove spurious SGID bit on directories ([#9447](https://github.com/opensearch-project/OpenSearch/pull/9447))
+
+### Fixed
+- Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725))
+- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045))
+- Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082))
+- Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089))
+- Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
+- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194))
+- Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256))
+- Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
+- Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))

From 1447d75e5e9ea042dd4bb645d71f25aea1fb42a8 Mon Sep 17 00:00:00 2001
From: Michael Froh 
Date: Thu, 12 Oct 2023 15:30:02 -0700
Subject: [PATCH 156/388] Point deprecated settings to the new location.
 (#10522)

I was looking through some cluster settings, stumbled across the search
back-pressure settings, saw they were deprecated, and the Javadoc
alluded to some myserious new settings.

I'm adding links to the class with the new settings to help future
developers.

Signed-off-by: Michael Froh 
---
 .../backpressure/settings/SearchBackpressureSettings.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
index 4c28d96d8289e..79494eb0d3c24 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java
@@ -55,7 +55,7 @@ private static class Defaults {
      * Defines the percentage of tasks to cancel relative to the number of successful task completions.
      * In other words, it is the number of tokens added to the bucket on each successful task completion.
      * 
                              
-     * The setting below is deprecated.
+     * The setting below is deprecated. The new setting is in {@link SearchShardTaskSettings}.
      * To keep backwards compatibility, the old usage is remained, and it's also used as the fallback for the new usage.
      */
     public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting(
@@ -72,7 +72,7 @@ private static class Defaults {
      * Defines the number of tasks to cancel per unit time (in millis).
      * In other words, it is the number of tokens added to the bucket each millisecond.
      * 
                              
-     * The setting below is deprecated.
+     * The setting below is deprecated. The new setting is in {@link SearchShardTaskSettings}.
      * To keep backwards compatibility, the old usage is remained, and it's also used as the fallback for the new usage.
      */
     public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting(
@@ -87,7 +87,7 @@ private static class Defaults {
     /**
      * Defines the maximum number of tasks that can be cancelled before being rate-limited.
      * 
                              
-     * The setting below is deprecated.
+     * The setting below is deprecated. The new setting is in {@link SearchShardTaskSettings}.
      * To keep backwards compatibility, the old usage is remained, and it's also used as the fallback for the new usage.
      */
     public static final Setting SETTING_CANCELLATION_BURST = Setting.doubleSetting(

From 25f2b769f85c24258e2c49cc0d6b09d6b2500b99 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Fri, 13 Oct 2023 17:03:05 -0400
Subject: [PATCH 157/388] Bump OpenTelemetry from 1.30.1 to 1.31.0 (#10617)

Signed-off-by: Andriy Redko 
---
 CHANGELOG.md                                                   | 1 +
 buildSrc/version.properties                                    | 3 ++-
 plugins/telemetry-otel/build.gradle                            | 2 +-
 .../telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1  | 1 -
 .../telemetry-otel/licenses/opentelemetry-api-1.31.0.jar.sha1  | 1 +
 .../licenses/opentelemetry-context-1.30.1.jar.sha1             | 1 -
 .../licenses/opentelemetry-context-1.31.0.jar.sha1             | 1 +
 .../licenses/opentelemetry-exporter-common-1.30.1.jar.sha1     | 1 -
 .../licenses/opentelemetry-exporter-common-1.31.0.jar.sha1     | 1 +
 .../licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1    | 1 -
 .../licenses/opentelemetry-exporter-logging-1.31.0.jar.sha1    | 1 +
 .../licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1       | 1 -
 .../licenses/opentelemetry-exporter-otlp-1.31.0.jar.sha1       | 1 +
 .../opentelemetry-exporter-otlp-common-1.30.1.jar.sha1         | 1 -
 .../opentelemetry-exporter-otlp-common-1.31.0.jar.sha1         | 1 +
 .../opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1       | 1 -
 .../opentelemetry-exporter-sender-okhttp-1.31.0.jar.sha1       | 1 +
 .../opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1    | 1 -
 .../opentelemetry-extension-incubator-1.31.0-alpha.jar.sha1    | 1 +
 .../telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1  | 1 -
 .../telemetry-otel/licenses/opentelemetry-sdk-1.31.0.jar.sha1  | 1 +
 .../licenses/opentelemetry-sdk-common-1.30.1.jar.sha1          | 1 -
 .../licenses/opentelemetry-sdk-common-1.31.0.jar.sha1          | 1 +
 .../licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1            | 1 -
 .../licenses/opentelemetry-sdk-logs-1.31.0.jar.sha1            | 1 +
 .../licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1         | 1 -
 .../licenses/opentelemetry-sdk-metrics-1.31.0.jar.sha1         | 1 +
 .../licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1           | 1 -
 .../licenses/opentelemetry-sdk-trace-1.31.0.jar.sha1           | 1 +
 .../licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1       | 1 +
 .../licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1       | 1 -
 .../org/opensearch/telemetry/tracing/OTelResourceProvider.java | 2 +-
 32 files changed, 19 insertions(+), 17 deletions(-)
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.31.0-alpha.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.31.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.31.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a9e5bb3982708..51f477fc38da8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963))
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
+- Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 64f9f9a8828ad..a5171aa582a86 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -68,4 +68,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.30.1
+opentelemetry         = 1.31.0
+opentelemetrysemconv  = 1.21.0-alpha
diff --git a/plugins/telemetry-otel/build.gradle b/plugins/telemetry-otel/build.gradle
index 04fff20947b4f..f5c367cb7643b 100644
--- a/plugins/telemetry-otel/build.gradle
+++ b/plugins/telemetry-otel/build.gradle
@@ -28,7 +28,7 @@ dependencies {
   api "io.opentelemetry:opentelemetry-sdk-trace:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-sdk-metrics:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-exporter-logging:${versions.opentelemetry}"
-  api "io.opentelemetry:opentelemetry-semconv:${versions.opentelemetry}-alpha"
+  api "io.opentelemetry.semconv:opentelemetry-semconv:${versions.opentelemetrysemconv}"
   api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-exporter-otlp:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-exporter-common:${versions.opentelemetry}"
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1
deleted file mode 100644
index b0ce00e191830..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-api-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a32dfbd7f01de6711fd0e970f8d4b4c0405056d6
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..eae141a8d1a23
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-api-1.31.0.jar.sha1
@@ -0,0 +1 @@
+bb24a44d73484c681c236aed84fe6c28d17f30e2
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1
deleted file mode 100644
index 84cb60a2f7acb..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-context-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-58f665ff01ce6b964cdf0b8cb5cd1c196dfe94ce
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..6e42973adc581
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-context-1.31.0.jar.sha1
@@ -0,0 +1 @@
+b8004737f7a970124e36ac71fde8eb88423e8cee
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1
deleted file mode 100644
index eccb15f7b7c8e..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f299d336dba1039478497f37b273dfa764c6faef
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..b119468e7f88b
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.31.0.jar.sha1
@@ -0,0 +1 @@
+b7b4baf5f9af72d5eb8a231dfb114ae31c57150d
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1
deleted file mode 100644
index 40537a399ab14..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-58f1a09e89955e6145babf8bcdf80c95174eb817
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..8f653922d6418
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.31.0.jar.sha1
@@ -0,0 +1 @@
+260e5363dad83a0ae65c16ad6a3dd2914e0db201
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1
deleted file mode 100644
index e88b7514ee54d..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-15692246539571c41180aff2b55abe527b939a7b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..103da4720de96
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.31.0.jar.sha1
@@ -0,0 +1 @@
+b6454464425dfd81519070caeca3824558a2f1ae
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1
deleted file mode 100644
index 86937743208c6..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-947cf43a6411c4a323e14594431040a476ad43e8
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..3db07532ceea9
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.31.0.jar.sha1
@@ -0,0 +1 @@
+d8c22b6851bbc3dbf5d2387b9bde158ed5416ba4
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1
deleted file mode 100644
index 068926277253c..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9f3a14515500e4df260ce7b10a668237a95ac791
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..10d9b7cdfe3e3
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.31.0.jar.sha1
@@ -0,0 +1 @@
+dd209381d58cfe81a989e29c9ca26d97c8dabd7a
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1
deleted file mode 100644
index bde43937e82e4..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.30.1-alpha.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bfcea9bd71f97dd4e8a4f92c15ba5659fb07ff05
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.31.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.31.0-alpha.jar.sha1
new file mode 100644
index 0000000000000..162890965a6eb
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.31.0-alpha.jar.sha1
@@ -0,0 +1 @@
+6c9f5c063309d92b6dd28bff0667f54b63afd36f
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1
deleted file mode 100644
index d425ed61cc4cd..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4d15a9ea26e8e6ea93287a9f4ee02d91e5a74392
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..d6ce31a31cc6f
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.31.0.jar.sha1
@@ -0,0 +1 @@
+2b2093be08a09ac536292bf6cecf8129cc7fb191
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1
deleted file mode 100644
index 6b32d98b0f7c7..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8e437ba87004bb63069d04fb06beae65b98dd13a
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..8a6a9705d836d
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.31.0.jar.sha1
@@ -0,0 +1 @@
+f492528288236e097e12fc1c45963dd82c70d33c
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1
deleted file mode 100644
index 13ef6de11e82d..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5985d0950746ad12b49cc42c063f26ddfbcaaacb
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..37d79f5c573f7
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.31.0.jar.sha1
@@ -0,0 +1 @@
+a63a203d3dc6f8875f8c26b9e3b522dc9a3f6280
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1
deleted file mode 100644
index fc5aad9c9011e..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b12825541c5dae52a0fb35045c1b36df3ca8f632
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..80179e4808f50
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.31.0.jar.sha1
@@ -0,0 +1 @@
+47cc23762fae728d68e4fda1dfb71986ae0b8b3e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1
deleted file mode 100644
index ac522b765da05..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.30.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c5531fbc44178a7bcfeb7021ae80e70a7c43458
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.31.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.31.0.jar.sha1
new file mode 100644
index 0000000000000..fd917a58ba77c
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.31.0.jar.sha1
@@ -0,0 +1 @@
+a3941197cfb8ae9eb9e482073480c0c3918b746c
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1
new file mode 100644
index 0000000000000..77b12c99464f6
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1
@@ -0,0 +1 @@
+207660e74d1e155272e9559fd4d27854b92fc6ac
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1
deleted file mode 100644
index 089a2484dd1d5..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.30.1-alpha.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8e8f7a97a4896a81846553275b9d61885be7ef50
\ No newline at end of file
diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java
index a6a1f12aab8a9..14a19f122c17b 100644
--- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java
+++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java
@@ -30,7 +30,7 @@
 import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
 import io.opentelemetry.sdk.trace.samplers.Sampler;
-import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
+import io.opentelemetry.semconv.ResourceAttributes;
 
 import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_BATCH_SIZE_SETTING;
 import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING;

From daf1350888f878868748172f576a0cdb3dc64b33 Mon Sep 17 00:00:00 2001
From: David Z <38449481+dzane17@users.noreply.github.com>
Date: Fri, 13 Oct 2023 14:12:46 -0700
Subject: [PATCH 158/388] Per request phase latency (#10351)

Signed-off-by: David Zane 
---
 CHANGELOG.md                                  |   1 +
 .../search/AbstractSearchAsyncAction.java     |   2 +
 .../action/search/MultiSearchRequest.java     |   5 +
 .../action/search/SearchRequest.java          |  31 +++-
 .../action/search/SearchResponse.java         | 132 +++++++++++++++++-
 .../action/search/SearchResponseMerger.java   |   1 +
 .../action/search/TransportSearchAction.java  | 129 +++++++++++++----
 .../common/settings/ClusterSettings.java      |   1 +
 .../rest/action/search/RestSearchAction.java  |   6 +
 .../AbstractSearchAsyncActionTests.java       |  15 +-
 .../action/search/SearchRequestTests.java     |   1 +
 .../action/search/SearchResponseTests.java    |  47 ++++++-
 .../search/SearchTimeProviderTests.java       |  54 +++++++
 .../search/RandomSearchRequestGenerator.java  |   3 +
 14 files changed, 394 insertions(+), 34 deletions(-)
 create mode 100644 server/src/test/java/org/opensearch/action/search/SearchTimeProviderTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 51f477fc38da8..a53d31c7861ff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
 
 ### Dependencies
 
diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
index 1c0a1280ad550..14f57218ae1dc 100644
--- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
@@ -215,6 +215,7 @@ public final void start() {
                     0,
                     0,
                     buildTookInMillis(),
+                    timeProvider.getPhaseTook(),
                     ShardSearchFailure.EMPTY_ARRAY,
                     clusters,
                     null
@@ -662,6 +663,7 @@ protected final SearchResponse buildSearchResponse(
             successfulOps.get(),
             skippedOps.get(),
             buildTookInMillis(),
+            timeProvider.getPhaseTook(),
             failures,
             clusters,
             searchContextId
diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
index da8f8f144eaf2..00e0345062d1c 100644
--- a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
@@ -277,6 +277,8 @@ public static void readMultiLineFormat(
                         } else if ("cancel_after_time_interval".equals(entry.getKey())
                             || "cancelAfterTimeInterval".equals(entry.getKey())) {
                                 searchRequest.setCancelAfterTimeInterval(nodeTimeValue(value, null));
+                            } else if ("phase_took".equals(entry.getKey())) {
+                                searchRequest.setPhaseTook(nodeBooleanValue(value));
                             } else {
                                 throw new IllegalArgumentException("key [" + entry.getKey() + "] is not supported in the metadata section");
                             }
@@ -374,6 +376,9 @@ public static void writeSearchRequestParams(SearchRequest request, XContentBuild
         if (request.getCancelAfterTimeInterval() != null) {
             xContentBuilder.field("cancel_after_time_interval", request.getCancelAfterTimeInterval().getStringRep());
         }
+        if (request.isPhaseTook() != null) {
+            xContentBuilder.field("phase_took", request.isPhaseTook());
+        }
         xContentBuilder.endObject();
     }
 
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
index 21cf0ed97b9da..9e50213eab5f9 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
@@ -117,6 +117,8 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
 
     private String pipeline;
 
+    private Boolean phaseTook = null;
+
     public SearchRequest() {
         this.localClusterAlias = null;
         this.absoluteStartMillis = DEFAULT_ABSOLUTE_START_MILLIS;
@@ -209,6 +211,7 @@ private SearchRequest(
         this.absoluteStartMillis = absoluteStartMillis;
         this.finalReduce = finalReduce;
         this.cancelAfterTimeInterval = searchRequest.cancelAfterTimeInterval;
+        this.phaseTook = searchRequest.phaseTook;
     }
 
     /**
@@ -253,6 +256,9 @@ public SearchRequest(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_2_7_0)) {
             pipeline = in.readOptionalString();
         }
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            phaseTook = in.readOptionalBoolean();
+        }
     }
 
     @Override
@@ -284,6 +290,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_7_0)) {
             out.writeOptionalString(pipeline);
         }
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalBoolean(phaseTook);
+        }
     }
 
     @Override
@@ -615,6 +624,20 @@ public void setPreFilterShardSize(int preFilterShardSize) {
         this.preFilterShardSize = preFilterShardSize;
     }
 
+    /**
+     * Returns value of user-provided phase_took query parameter for this search request.
+     */
+    public Boolean isPhaseTook() {
+        return phaseTook;
+    }
+
+    /**
+     * Sets value of phase_took query param if provided by user. Defaults to null.
+     */
+    public void setPhaseTook(Boolean phaseTook) {
+        this.phaseTook = phaseTook;
+    }
+
     /**
      * Returns a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
      * the search request expands to exceeds the threshold, or null if the threshold is unspecified.
@@ -719,7 +742,8 @@ public boolean equals(Object o) {
             && absoluteStartMillis == that.absoluteStartMillis
             && ccsMinimizeRoundtrips == that.ccsMinimizeRoundtrips
             && Objects.equals(cancelAfterTimeInterval, that.cancelAfterTimeInterval)
-            && Objects.equals(pipeline, that.pipeline);
+            && Objects.equals(pipeline, that.pipeline)
+            && Objects.equals(phaseTook, that.phaseTook);
     }
 
     @Override
@@ -740,7 +764,8 @@ public int hashCode() {
             localClusterAlias,
             absoluteStartMillis,
             ccsMinimizeRoundtrips,
-            cancelAfterTimeInterval
+            cancelAfterTimeInterval,
+            phaseTook
         );
     }
 
@@ -783,6 +808,8 @@ public String toString() {
             + cancelAfterTimeInterval
             + ", pipeline="
             + pipeline
+            + ", phaseTook="
+            + phaseTook
             + "}";
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponse.java b/server/src/main/java/org/opensearch/action/search/SearchResponse.java
index a546311a1f668..91f0dc0737637 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.search;
 
 import org.apache.lucene.search.TotalHits;
+import org.opensearch.Version;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.StatusToXContentObject;
@@ -63,7 +64,9 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Supplier;
@@ -94,6 +97,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
     private final ShardSearchFailure[] shardFailures;
     private final Clusters clusters;
     private final long tookInMillis;
+    private final PhaseTook phaseTook;
 
     public SearchResponse(StreamInput in) throws IOException {
         super(in);
@@ -112,6 +116,11 @@ public SearchResponse(StreamInput in) throws IOException {
         clusters = new Clusters(in);
         scrollId = in.readOptionalString();
         tookInMillis = in.readVLong();
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            phaseTook = in.readOptionalWriteable(PhaseTook::new);
+        } else {
+            phaseTook = null;
+        }
         skippedShards = in.readVInt();
         pointInTimeId = in.readOptionalString();
     }
@@ -126,7 +135,32 @@ public SearchResponse(
         ShardSearchFailure[] shardFailures,
         Clusters clusters
     ) {
-        this(internalResponse, scrollId, totalShards, successfulShards, skippedShards, tookInMillis, shardFailures, clusters, null);
+        this(internalResponse, scrollId, totalShards, successfulShards, skippedShards, tookInMillis, null, shardFailures, clusters, null);
+    }
+
+    public SearchResponse(
+        SearchResponseSections internalResponse,
+        String scrollId,
+        int totalShards,
+        int successfulShards,
+        int skippedShards,
+        long tookInMillis,
+        ShardSearchFailure[] shardFailures,
+        Clusters clusters,
+        String pointInTimeId
+    ) {
+        this(
+            internalResponse,
+            scrollId,
+            totalShards,
+            successfulShards,
+            skippedShards,
+            tookInMillis,
+            null,
+            shardFailures,
+            clusters,
+            pointInTimeId
+        );
     }
 
     public SearchResponse(
@@ -136,6 +170,7 @@ public SearchResponse(
         int successfulShards,
         int skippedShards,
         long tookInMillis,
+        PhaseTook phaseTook,
         ShardSearchFailure[] shardFailures,
         Clusters clusters,
         String pointInTimeId
@@ -148,6 +183,7 @@ public SearchResponse(
         this.successfulShards = successfulShards;
         this.skippedShards = skippedShards;
         this.tookInMillis = tookInMillis;
+        this.phaseTook = phaseTook;
         this.shardFailures = shardFailures;
         assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards;
         assert scrollId == null || pointInTimeId == null : "SearchResponse can't have both scrollId ["
@@ -210,6 +246,13 @@ public TimeValue getTook() {
         return new TimeValue(tookInMillis);
     }
 
+    /**
+     * How long the request took in each search phase.
+     */
+    public PhaseTook getPhaseTook() {
+        return phaseTook;
+    }
+
     /**
      * The total number of shards the search was executed on.
      */
@@ -298,6 +341,9 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
             builder.field(POINT_IN_TIME_ID.getPreferredName(), pointInTimeId);
         }
         builder.field(TOOK.getPreferredName(), tookInMillis);
+        if (phaseTook != null) {
+            phaseTook.toXContent(builder, params);
+        }
         builder.field(TIMED_OUT.getPreferredName(), isTimedOut());
         if (isTerminatedEarly() != null) {
             builder.field(TERMINATED_EARLY.getPreferredName(), isTerminatedEarly());
@@ -337,6 +383,7 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
         Boolean terminatedEarly = null;
         int numReducePhases = 1;
         long tookInMillis = -1;
+        PhaseTook phaseTook = null;
         int successfulShards = -1;
         int totalShards = -1;
         int skippedShards = 0; // 0 for BWC
@@ -401,6 +448,24 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
                             parser.skipChildren();
                         }
                     }
+                } else if (PhaseTook.PHASE_TOOK.match(currentFieldName, parser.getDeprecationHandler())) {
+                    Map phaseTookMap = new HashMap<>();
+
+                    while ((token = parser.nextToken()) != Token.END_OBJECT) {
+                        if (token == Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (token.isValue()) {
+                            try {
+                                SearchPhaseName.valueOf(currentFieldName.toUpperCase(Locale.ROOT));
+                                phaseTookMap.put(currentFieldName, parser.longValue());
+                            } catch (final IllegalArgumentException ex) {
+                                parser.skipChildren();
+                            }
+                        } else {
+                            parser.skipChildren();
+                        }
+                    }
+                    phaseTook = new PhaseTook(phaseTookMap);
                 } else if (Clusters._CLUSTERS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     int successful = -1;
                     int total = -1;
@@ -472,6 +537,7 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE
             successfulShards,
             skippedShards,
             tookInMillis,
+            phaseTook,
             failures.toArray(ShardSearchFailure.EMPTY_ARRAY),
             clusters,
             searchContextId
@@ -491,6 +557,9 @@ public void writeTo(StreamOutput out) throws IOException {
         clusters.writeTo(out);
         out.writeOptionalString(scrollId);
         out.writeVLong(tookInMillis);
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalWriteable(phaseTook);
+        }
         out.writeVInt(skippedShards);
         out.writeOptionalString(pointInTimeId);
     }
@@ -604,6 +673,67 @@ public String toString() {
         }
     }
 
+    /**
+     * Holds info about the clusters that the search was executed on: how many in total, how many of them were successful
+     * and how many of them were skipped.
+     *
+     * @opensearch.internal
+     */
+    public static class PhaseTook implements ToXContentFragment, Writeable {
+        static final ParseField PHASE_TOOK = new ParseField("phase_took");
+        private final Map phaseTookMap;
+
+        public PhaseTook(Map phaseTookMap) {
+            this.phaseTookMap = phaseTookMap;
+        }
+
+        private PhaseTook(StreamInput in) throws IOException {
+            this(in.readMap(StreamInput::readString, StreamInput::readLong));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeMap(phaseTookMap, StreamOutput::writeString, StreamOutput::writeLong);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(PHASE_TOOK.getPreferredName());
+
+            for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) {
+                if (phaseTookMap.containsKey(searchPhaseName.getName())) {
+                    builder.field(searchPhaseName.getName(), phaseTookMap.get(searchPhaseName.getName()));
+                } else {
+                    builder.field(searchPhaseName.getName(), 0);
+                }
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            PhaseTook phaseTook = (PhaseTook) o;
+
+            if (phaseTook.phaseTookMap.equals(phaseTookMap)) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(phaseTookMap);
+        }
+    }
+
     static SearchResponse empty(Supplier tookInMillisSupplier, Clusters clusters) {
         SearchHits searchHits = new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), Float.NaN);
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(
diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java
index f90e98106f93f..054bd578cc56c 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java
@@ -236,6 +236,7 @@ SearchResponse getMergedResponse(SearchResponse.Clusters clusters) {
             successfulShards,
             skippedShards,
             tookInMillis,
+            searchTimeProvider.getPhaseTook(),
             shardFailures,
             clusters,
             null
diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
index cff1005beff27..284f71bd9da62 100644
--- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
@@ -98,6 +98,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -154,6 +155,14 @@ public class TransportSearchAction extends HandledTransportAction SEARCH_PHASE_TOOK_ENABLED = Setting.boolSetting(
+        SEARCH_PHASE_TOOK_ENABLED_KEY,
+        false,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
     private final NodeClient client;
     private final ThreadPool threadPool;
     private final ClusterService clusterService;
@@ -252,6 +261,8 @@ private Map resolveIndexBoosts(SearchRequest searchRequest, Clust
     }
 
     /**
+     * Listener to track request-level tookTime and phase tookTimes from the coordinator.
+     *
      * Search operations need two clocks. One clock is to fulfill real clock needs (e.g., resolving
      * "now" to an index name). Another clock is needed for measuring how long a search operation
      * took. These two uses are at odds with each other. There are many issues with using a real
@@ -261,11 +272,12 @@ private Map resolveIndexBoosts(SearchRequest searchRequest, Clust
      *
      * @opensearch.internal
      */
-    static final class SearchTimeProvider {
+    static final class SearchTimeProvider implements SearchRequestOperationsListener {
 
         private final long absoluteStartMillis;
         private final long relativeStartNanos;
         private final LongSupplier relativeCurrentNanosProvider;
+        private boolean phaseTook = false;
 
         /**
          * Instantiates a new search time provider. The absolute start time is the real clock time
@@ -291,6 +303,47 @@ long getAbsoluteStartMillis() {
         long buildTookInMillis() {
             return TimeUnit.NANOSECONDS.toMillis(relativeCurrentNanosProvider.getAsLong() - relativeStartNanos);
         }
+
+        public void setPhaseTook(boolean phaseTook) {
+            this.phaseTook = phaseTook;
+        }
+
+        public boolean isPhaseTook() {
+            return phaseTook;
+        }
+
+        SearchResponse.PhaseTook getPhaseTook() {
+            if (phaseTook) {
+                Map phaseTookMap = new HashMap<>();
+                // Convert Map to Map for SearchResponse()
+                for (SearchPhaseName searchPhaseName : phaseStatsMap.keySet()) {
+                    phaseTookMap.put(searchPhaseName.getName(), phaseStatsMap.get(searchPhaseName));
+                }
+                return new SearchResponse.PhaseTook(phaseTookMap);
+            } else {
+                return null;
+            }
+        }
+
+        Map phaseStatsMap = new EnumMap<>(SearchPhaseName.class);
+
+        @Override
+        public void onPhaseStart(SearchPhaseContext context) {}
+
+        @Override
+        public void onPhaseEnd(SearchPhaseContext context) {
+            phaseStatsMap.put(
+                context.getCurrentPhase().getSearchPhaseName(),
+                TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - context.getCurrentPhase().getStartTimeInNanos())
+            );
+        }
+
+        @Override
+        public void onPhaseFailure(SearchPhaseContext context) {}
+
+        public Long getPhaseTookTime(SearchPhaseName searchPhaseName) {
+            return phaseStatsMap.get(searchPhaseName);
+        }
     }
 
     @Override
@@ -332,13 +385,6 @@ public void executeRequest(
         SinglePhaseSearchAction phaseSearchAction,
         ActionListener listener
     ) {
-        final List searchListenersList = createSearchListenerList();
-        final SearchRequestOperationsListener searchRequestOperationsListener;
-        if (!CollectionUtils.isEmpty(searchListenersList)) {
-            searchRequestOperationsListener = new SearchRequestOperationsListener.CompositeListener(searchListenersList, logger);
-        } else {
-            searchRequestOperationsListener = null;
-        }
         executeRequest(task, searchRequest, new SearchAsyncActionProvider() {
             @Override
             public AbstractSearchAsyncAction asyncSearchAction(
@@ -355,7 +401,8 @@ public AbstractSearchAsyncAction asyncSearchAction(
                 ActionListener listener,
                 boolean preFilter,
                 ThreadPool threadPool,
-                SearchResponse.Clusters clusters
+                SearchResponse.Clusters clusters,
+                SearchRequestOperationsListener searchRequestOperationsListener
             ) {
                 return new AbstractSearchAsyncAction(
                     actionName,
@@ -419,6 +466,16 @@ private void executeRequest(
             relativeStartNanos,
             System::nanoTime
         );
+
+        final List searchListenersList = createSearchListenerList(originalSearchRequest, timeProvider);
+
+        final SearchRequestOperationsListener searchRequestOperationsListener;
+        if (!CollectionUtils.isEmpty(searchListenersList)) {
+            searchRequestOperationsListener = new SearchRequestOperationsListener.CompositeListener(searchListenersList, logger);
+        } else {
+            searchRequestOperationsListener = null;
+        }
+
         PipelinedRequest searchRequest;
         ActionListener listener;
         try {
@@ -462,7 +519,8 @@ private void executeRequest(
                     clusterState,
                     listener,
                     searchContext,
-                    searchAsyncActionProvider
+                    searchAsyncActionProvider,
+                    searchRequestOperationsListener
                 );
             } else {
                 if (shouldMinimizeRoundtrips(searchRequest)) {
@@ -483,7 +541,8 @@ private void executeRequest(
                             clusterState,
                             l,
                             searchContext,
-                            searchAsyncActionProvider
+                            searchAsyncActionProvider,
+                            searchRequestOperationsListener
                         )
                     );
                 } else {
@@ -533,7 +592,8 @@ private void executeRequest(
                                 listener,
                                 new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()),
                                 searchContext,
-                                searchAsyncActionProvider
+                                searchAsyncActionProvider,
+                                searchRequestOperationsListener
                             );
                         }, listener::onFailure)
                     );
@@ -622,6 +682,7 @@ public void onResponse(SearchResponse searchResponse) {
                             searchResponse.getSuccessfulShards(),
                             searchResponse.getSkippedShards(),
                             timeProvider.buildTookInMillis(),
+                            timeProvider.getPhaseTook(),
                             searchResponse.getShardFailures(),
                             new SearchResponse.Clusters(1, 1, 0),
                             searchResponse.pointInTimeId()
@@ -811,7 +872,8 @@ private void executeLocalSearch(
         ClusterState clusterState,
         ActionListener listener,
         SearchContextId searchContext,
-        SearchAsyncActionProvider searchAsyncActionProvider
+        SearchAsyncActionProvider searchAsyncActionProvider,
+        SearchRequestOperationsListener searchRequestOperationsListener
     ) {
         executeSearch(
             (SearchTask) task,
@@ -825,7 +887,8 @@ private void executeLocalSearch(
             listener,
             SearchResponse.Clusters.EMPTY,
             searchContext,
-            searchAsyncActionProvider
+            searchAsyncActionProvider,
+            searchRequestOperationsListener
         );
     }
 
@@ -943,7 +1006,8 @@ private void executeSearch(
         ActionListener listener,
         SearchResponse.Clusters clusters,
         @Nullable SearchContextId searchContext,
-        SearchAsyncActionProvider searchAsyncActionProvider
+        SearchAsyncActionProvider searchAsyncActionProvider,
+        SearchRequestOperationsListener searchRequestOperationsListener
     ) {
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
@@ -1044,7 +1108,8 @@ private void executeSearch(
             listener,
             preFilterSearchShards,
             threadPool,
-            clusters
+            clusters,
+            searchRequestOperationsListener
         ).start();
     }
 
@@ -1127,15 +1192,30 @@ AbstractSearchAsyncAction asyncSearchAction(
             ActionListener listener,
             boolean preFilter,
             ThreadPool threadPool,
-            SearchResponse.Clusters clusters
+            SearchResponse.Clusters clusters,
+            SearchRequestOperationsListener searchRequestOperationsListener
         );
     }
 
-    private List createSearchListenerList() {
+    private List createSearchListenerList(SearchRequest searchRequest, SearchTimeProvider timeProvider) {
         final List searchListenersList = new ArrayList<>();
+
         if (isRequestStatsEnabled) {
             searchListenersList.add(searchRequestStats);
         }
+
+        // phase_took is enabled with request param and/or cluster setting
+        Boolean phaseTookRequestParam = searchRequest.isPhaseTook();
+        if (phaseTookRequestParam == null) {    // check cluster setting only when request param is undefined
+            if (clusterService.getClusterSettings().get(TransportSearchAction.SEARCH_PHASE_TOOK_ENABLED)) {
+                timeProvider.setPhaseTook(true);
+                searchListenersList.add(timeProvider);
+            }
+        } else if (phaseTookRequestParam == true) {
+            timeProvider.setPhaseTook(true);
+            searchListenersList.add(timeProvider);
+        }
+
         return searchListenersList;
     }
 
@@ -1153,15 +1233,9 @@ private AbstractSearchAsyncAction searchAsyncAction
         ActionListener listener,
         boolean preFilter,
         ThreadPool threadPool,
-        SearchResponse.Clusters clusters
+        SearchResponse.Clusters clusters,
+        SearchRequestOperationsListener searchRequestOperationsListener
     ) {
-        final List searchListenersList = createSearchListenerList();
-        final SearchRequestOperationsListener searchRequestOperationsListener;
-        if (!CollectionUtils.isEmpty(searchListenersList)) {
-            searchRequestOperationsListener = new SearchRequestOperationsListener.CompositeListener(searchListenersList, logger);
-        } else {
-            searchRequestOperationsListener = null;
-        }
         if (preFilter) {
             return new CanMatchPreFilterSearchPhase(
                 logger,
@@ -1192,7 +1266,8 @@ private AbstractSearchAsyncAction searchAsyncAction
                         listener,
                         false,
                         threadPool,
-                        clusters
+                        clusters,
+                        searchRequestOperationsListener
                     );
                     return new SearchPhase(action.getName()) {
                         @Override
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 4cd3490cffb4c..ad2b89aa3948d 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -375,6 +375,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 TransportSearchAction.SHARD_COUNT_LIMIT_SETTING,
                 TransportSearchAction.SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING,
                 TransportSearchAction.SEARCH_REQUEST_STATS_ENABLED,
+                TransportSearchAction.SEARCH_PHASE_TOOK_ENABLED,
                 RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE,
                 SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER,
                 RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
index ebfd082d974fd..080366e536da1 100644
--- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
@@ -180,6 +180,12 @@ public static void parseSearchRequest(
             searchRequest.allowPartialSearchResults(request.paramAsBoolean("allow_partial_search_results", null));
         }
 
+        if (request.hasParam("phase_took")) {
+            // only set if we have the parameter passed to override the cluster-level default
+            // else phaseTook = null
+            searchRequest.setPhaseTook(request.paramAsBoolean("phase_took", true));
+        }
+
         // do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types
         // from the REST layer. these modes are an internal optimization and should
         // not be specified explicitly by the user.
diff --git a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java
index f628bb3201452..edac50813e191 100644
--- a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java
@@ -688,7 +688,11 @@ private SearchDfsQueryThenFetchAsyncAction createSearchDfsQueryThenFetchAsyncAct
         );
         AtomicReference exception = new AtomicReference<>();
         ActionListener listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
-
+        TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(
+            0,
+            System.nanoTime(),
+            System::nanoTime
+        );
         return new SearchDfsQueryThenFetchAsyncAction(
             logger,
             null,
@@ -702,7 +706,7 @@ private SearchDfsQueryThenFetchAsyncAction createSearchDfsQueryThenFetchAsyncAct
             searchRequest,
             listener,
             shardsIter,
-            null,
+            timeProvider,
             null,
             task,
             SearchResponse.Clusters.EMPTY,
@@ -734,6 +738,11 @@ private SearchQueryThenFetchAsyncAction createSearchQueryThenFetchAsyncAction(
         );
         AtomicReference exception = new AtomicReference<>();
         ActionListener listener = ActionListener.wrap(response -> fail("onResponse should not be called"), exception::set);
+        TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(
+            0,
+            System.nanoTime(),
+            System::nanoTime
+        );
         return new SearchQueryThenFetchAsyncAction(
             logger,
             null,
@@ -747,7 +756,7 @@ private SearchQueryThenFetchAsyncAction createSearchQueryThenFetchAsyncAction(
             searchRequest,
             listener,
             shardsIter,
-            null,
+            timeProvider,
             null,
             task,
             SearchResponse.Clusters.EMPTY,
diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
index 25d8c5551880f..cdd0ea863ce37 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
@@ -244,6 +244,7 @@ private SearchRequest mutate(SearchRequest searchRequest) {
         );
         mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder)));
         mutators.add(() -> mutation.setCcsMinimizeRoundtrips(searchRequest.isCcsMinimizeRoundtrips() == false));
+        mutators.add(() -> mutation.setPhaseTook(searchRequest.isPhaseTook() == false));
         mutators.add(
             () -> mutation.setCancelAfterTimeInterval(
                 searchRequest.getCancelAfterTimeInterval() != null
diff --git a/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java b/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java
index 097e922147698..c9e59ab4ea04d 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchResponseTests.java
@@ -74,7 +74,9 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 
 import static java.util.Collections.singletonMap;
@@ -152,6 +154,11 @@ public SearchResponse createTestItem(
         Boolean terminatedEarly = randomBoolean() ? null : randomBoolean();
         int numReducePhases = randomIntBetween(1, 10);
         long tookInMillis = randomNonNegativeLong();
+        Map phaseTookMap = new HashMap<>();
+        for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) {
+            phaseTookMap.put(searchPhaseName.getName(), randomNonNegativeLong());
+        }
+        SearchResponse.PhaseTook phaseTook = new SearchResponse.PhaseTook(phaseTookMap);
         int totalShards = randomIntBetween(1, Integer.MAX_VALUE);
         int successfulShards = randomIntBetween(0, totalShards);
         int skippedShards = randomIntBetween(0, totalShards);
@@ -182,6 +189,7 @@ public SearchResponse createTestItem(
             successfulShards,
             skippedShards,
             tookInMillis,
+            phaseTook,
             shardSearchFailures,
             randomBoolean() ? randomClusters() : SearchResponse.Clusters.EMPTY,
             null
@@ -353,6 +361,14 @@ public void testToXContent() {
             assertEquals(1, searchExtBuilders.size());
         }
         {
+            Map phaseTookMap = new HashMap<>();
+            for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) {
+                phaseTookMap.put(searchPhaseName.getName(), 0L);
+            }
+            phaseTookMap.put(SearchPhaseName.QUERY.getName(), 50L);
+            phaseTookMap.put(SearchPhaseName.FETCH.getName(), 25L);
+            phaseTookMap.put(SearchPhaseName.EXPAND.getName(), 30L);
+            SearchResponse.PhaseTook phaseTook = new SearchResponse.PhaseTook(phaseTookMap);
             SearchResponse response = new SearchResponse(
                 new InternalSearchResponse(
                     new SearchHits(hits, new TotalHits(100, TotalHits.Relation.EQUAL_TO), 1.5f),
@@ -368,13 +384,24 @@ public void testToXContent() {
                 0,
                 0,
                 0,
+                phaseTook,
                 ShardSearchFailure.EMPTY_ARRAY,
-                new SearchResponse.Clusters(5, 3, 2)
+                new SearchResponse.Clusters(5, 3, 2),
+                null
             );
             StringBuilder expectedString = new StringBuilder();
             expectedString.append("{");
             {
                 expectedString.append("\"took\":0,");
+                expectedString.append("\"phase_took\":");
+                {
+                    expectedString.append("{\"dfs_pre_query\":0,");
+                    expectedString.append("\"query\":50,");
+                    expectedString.append("\"fetch\":25,");
+                    expectedString.append("\"dfs_query\":0,");
+                    expectedString.append("\"expand\":30,");
+                    expectedString.append("\"can_match\":0},");
+                }
                 expectedString.append("\"timed_out\":false,");
                 expectedString.append("\"_shards\":");
                 {
@@ -477,6 +504,24 @@ public void testToXContentEmptyClusters() throws IOException {
         assertEquals(0, builder.toString().length());
     }
 
+    public void testSearchResponsePhaseTookEquals() throws IOException {
+        SearchResponse.PhaseTook phaseTookA = new SearchResponse.PhaseTook(Map.of("foo", 0L, "bar", 1L));
+        SearchResponse.PhaseTook phaseTookB = new SearchResponse.PhaseTook(Map.of("foo", 1L, "bar", 1L));
+        SearchResponse.PhaseTook phaseTookC = new SearchResponse.PhaseTook(Map.of("foo", 0L));
+        SearchResponse.PhaseTook phaseTookD = new SearchResponse.PhaseTook(Map.of());
+
+        assertNotEquals(phaseTookA, phaseTookB);
+        assertNotEquals(phaseTookB, phaseTookA);
+        assertNotEquals(phaseTookA, phaseTookC);
+        assertNotEquals(phaseTookC, phaseTookA);
+        assertNotEquals(phaseTookA, phaseTookD);
+        assertNotEquals(phaseTookD, phaseTookA);
+        assertEquals(phaseTookA, phaseTookA);
+        assertEquals(phaseTookB, phaseTookB);
+        assertEquals(phaseTookC, phaseTookC);
+        assertEquals(phaseTookD, phaseTookD);
+    }
+
     static class DummySearchExtBuilder extends SearchExtBuilder {
 
         static ParseField DUMMY_FIELD = new ParseField("dummy");
diff --git a/server/src/test/java/org/opensearch/action/search/SearchTimeProviderTests.java b/server/src/test/java/org/opensearch/action/search/SearchTimeProviderTests.java
new file mode 100644
index 0000000000000..f0f1a43e6c21e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/action/search/SearchTimeProviderTests.java
@@ -0,0 +1,54 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.search;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SearchTimeProviderTests extends OpenSearchTestCase {
+
+    public void testSearchTimeProviderPhaseFailure() {
+        TransportSearchAction.SearchTimeProvider testTimeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0);
+        SearchPhaseContext ctx = mock(SearchPhaseContext.class);
+        SearchPhase mockSearchPhase = mock(SearchPhase.class);
+        when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase);
+
+        for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) {
+            when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName);
+            testTimeProvider.onPhaseStart(ctx);
+            assertNull(testTimeProvider.getPhaseTookTime(searchPhaseName));
+            testTimeProvider.onPhaseFailure(ctx);
+            assertNull(testTimeProvider.getPhaseTookTime(searchPhaseName));
+        }
+    }
+
+    public void testSearchTimeProviderPhaseEnd() {
+        TransportSearchAction.SearchTimeProvider testTimeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0);
+
+        SearchPhaseContext ctx = mock(SearchPhaseContext.class);
+        SearchPhase mockSearchPhase = mock(SearchPhase.class);
+        when(ctx.getCurrentPhase()).thenReturn(mockSearchPhase);
+
+        for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) {
+            when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName);
+            long tookTimeInMillis = randomIntBetween(1, 100);
+            testTimeProvider.onPhaseStart(ctx);
+            long startTime = System.nanoTime() - TimeUnit.MILLISECONDS.toNanos(tookTimeInMillis);
+            when(mockSearchPhase.getStartTimeInNanos()).thenReturn(startTime);
+            assertNull(testTimeProvider.getPhaseTookTime(searchPhaseName));
+            testTimeProvider.onPhaseEnd(ctx);
+            assertThat(testTimeProvider.getPhaseTookTime(searchPhaseName), greaterThanOrEqualTo(tookTimeInMillis));
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java
index b942136e1f1e2..74de1e6d96d93 100644
--- a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java
+++ b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java
@@ -131,6 +131,9 @@ public static SearchRequest randomSearchRequest(Supplier ra
         if (randomBoolean()) {
             searchRequest.setCancelAfterTimeInterval(TimeValue.parseTimeValue(randomTimeValue(), null, "cancel_after_time_interval"));
         }
+        if (randomBoolean()) {
+            searchRequest.setPhaseTook(randomBoolean());
+        }
         return searchRequest;
     }
 

From 53be8f377a9de7b16da99def03b5e87f39cce430 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Fri, 13 Oct 2023 14:58:26 -0700
Subject: [PATCH 159/388] Fix broken test
 testCommitOnCloseThrowsException_decRefStore (#10621)

This test was incorrectly hardcoding a segment to delete to force corruption.
If a merge occurs randomly before segments are copied the expected segment did not exist failing the test.
This will now look for any .si and delete it.

Signed-off-by: Marc Handalian 
---
 .../opensearch/index/engine/NRTReplicationEngineTests.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java
index ee25d3789fb13..09484cd1b5840 100644
--- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java
@@ -35,6 +35,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicLong;
@@ -591,7 +592,9 @@ public void testCommitOnCloseThrowsException_decRefStore() throws Exception {
         indexOperations(nrtEngine, operations);
         // wipe the nrt directory initially so we can sync with primary.
         cleanAndCopySegmentsFromPrimary(nrtEngine);
-        nrtEngineStore.directory().deleteFile("_0.si");
+        final Optional toDelete = Set.of(nrtEngineStore.directory().listAll()).stream().filter(f -> f.endsWith(".si")).findAny();
+        assertTrue(toDelete.isPresent());
+        nrtEngineStore.directory().deleteFile(toDelete.get());
         assertEquals(2, nrtEngineStore.refCount());
         nrtEngine.close();
         assertEquals(1, nrtEngineStore.refCount());

From 6c1bd487da8ca1794f29302d7a6c6a713f9c6a01 Mon Sep 17 00:00:00 2001
From: Bhumika Saini 
Date: Sat, 14 Oct 2023 19:46:09 +0530
Subject: [PATCH 160/388] [Remote Store] Add Remote Store backpressure
 rejection stats to _nodes/stats (#10524)

Signed-off-by: Bhumika Saini 
---
 CHANGELOG.md                                  |  1 +
 .../index/remote/RemoteSegmentStats.java      | 51 ++++++++++++++++---
 .../remote/RemoteSegmentTransferTracker.java  |  3 +-
 .../cluster/node/stats/NodeStatsTests.java    |  1 +
 .../index/shard/IndexShardTests.java          |  4 ++
 5 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a53d31c7861ff..011e404c3d302 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -90,6 +90,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Dependencies
 
 ### Changed
+- [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
 
 ### Deprecated
 
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
index c7863536adf20..5992923a4157b 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.index.remote;
 
+import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -75,6 +76,10 @@ public class RemoteSegmentStats implements Writeable, ToXContentFragment {
      * Total time spent in downloading segments from remote store
      */
     private long totalDownloadTime;
+    /**
+     * Total rejections due to remote store upload backpressure
+     */
+    private long totalRejections;
 
     public RemoteSegmentStats() {}
 
@@ -90,6 +95,10 @@ public RemoteSegmentStats(StreamInput in) throws IOException {
         totalRefreshBytesLag = in.readLong();
         totalUploadTime = in.readLong();
         totalDownloadTime = in.readLong();
+        // TODO: change to V_2_12_0 on main after backport to 2.x
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            totalRejections = in.readVLong();
+        }
     }
 
     /**
@@ -115,6 +124,7 @@ public RemoteSegmentStats(RemoteSegmentTransferTracker.Stats trackerStats) {
         this.totalRefreshBytesLag = trackerStats.bytesLag;
         this.totalUploadTime = trackerStats.totalUploadTimeInMs;
         this.totalDownloadTime = trackerStats.directoryFileTransferTrackerStats.totalTransferTimeInMs;
+        this.totalRejections = trackerStats.rejectionCount;
     }
 
     // Getter and setters. All are visible for testing
@@ -207,6 +217,14 @@ public void addTotalDownloadTime(long totalDownloadTime) {
         this.totalDownloadTime += totalDownloadTime;
     }
 
+    public long getTotalRejections() {
+        return totalRejections;
+    }
+
+    public void addTotalRejections(long totalRejections) {
+        this.totalRejections += totalRejections;
+    }
+
     /**
      * Adds existing stats. Used for stats roll-ups at index or node level
      *
@@ -225,6 +243,7 @@ public void add(RemoteSegmentStats existingStats) {
             this.totalRefreshBytesLag += existingStats.getTotalRefreshBytesLag();
             this.totalUploadTime += existingStats.getTotalUploadTime();
             this.totalDownloadTime += existingStats.getTotalDownloadTime();
+            this.totalRejections += existingStats.totalRejections;
         }
     }
 
@@ -241,18 +260,26 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(totalRefreshBytesLag);
         out.writeLong(totalUploadTime);
         out.writeLong(totalDownloadTime);
+        // TODO: change to V_2_12_0 on main after backport to 2.x
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeVLong(totalRejections);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.REMOTE_STORE);
+
         builder.startObject(Fields.UPLOAD);
         buildUploadStats(builder);
-        builder.endObject();
+        builder.endObject(); // UPLOAD
+
         builder.startObject(Fields.DOWNLOAD);
         buildDownloadStats(builder);
-        builder.endObject();
-        builder.endObject();
+        builder.endObject(); // DOWNLOAD
+
+        builder.endObject(); // REMOTE_STORE
+
         return builder;
     }
 
@@ -261,13 +288,19 @@ private void buildUploadStats(XContentBuilder builder) throws IOException {
         builder.humanReadableField(Fields.STARTED_BYTES, Fields.STARTED, new ByteSizeValue(uploadBytesStarted));
         builder.humanReadableField(Fields.SUCCEEDED_BYTES, Fields.SUCCEEDED, new ByteSizeValue(uploadBytesSucceeded));
         builder.humanReadableField(Fields.FAILED_BYTES, Fields.FAILED, new ByteSizeValue(uploadBytesFailed));
-        builder.endObject();
+        builder.endObject(); // TOTAL_UPLOAD_SIZE
+
         builder.startObject(Fields.REFRESH_SIZE_LAG);
         builder.humanReadableField(Fields.TOTAL_BYTES, Fields.TOTAL, new ByteSizeValue(totalRefreshBytesLag));
         builder.humanReadableField(Fields.MAX_BYTES, Fields.MAX, new ByteSizeValue(maxRefreshBytesLag));
-        builder.endObject();
+        builder.endObject(); // REFRESH_SIZE_LAG
+
         builder.humanReadableField(Fields.MAX_REFRESH_TIME_LAG_IN_MILLIS, Fields.MAX_REFRESH_TIME_LAG, new TimeValue(maxRefreshTimeLag));
         builder.humanReadableField(Fields.TOTAL_TIME_SPENT_IN_MILLIS, Fields.TOTAL_TIME_SPENT, new TimeValue(totalUploadTime));
+
+        builder.startObject(Fields.PRESSURE);
+        builder.field(Fields.TOTAL_REJECTIONS, totalRejections);
+        builder.endObject(); // PRESSURE
     }
 
     private void buildDownloadStats(XContentBuilder builder) throws IOException {
@@ -300,6 +333,8 @@ static final class Fields {
         static final String MAX_BYTES = "max_bytes";
         static final String TOTAL_TIME_SPENT = "total_time_spent";
         static final String TOTAL_TIME_SPENT_IN_MILLIS = "total_time_spent_in_millis";
+        static final String PRESSURE = "pressure";
+        static final String TOTAL_REJECTIONS = "total_rejections";
     }
 
     @Override
@@ -318,7 +353,8 @@ public boolean equals(Object o) {
             && maxRefreshBytesLag == that.maxRefreshBytesLag
             && totalRefreshBytesLag == that.totalRefreshBytesLag
             && totalUploadTime == that.totalUploadTime
-            && totalDownloadTime == that.totalDownloadTime;
+            && totalDownloadTime == that.totalDownloadTime
+            && totalRejections == that.totalRejections;
     }
 
     @Override
@@ -334,7 +370,8 @@ public int hashCode() {
             maxRefreshBytesLag,
             totalRefreshBytesLag,
             totalUploadTime,
-            totalDownloadTime
+            totalDownloadTime,
+            totalRejections
         );
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
index 05081180bb179..2a703f17aa953 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
@@ -267,7 +267,8 @@ public long getRejectionCount() {
         return rejectionCount.get();
     }
 
-    void incrementRejectionCount() {
+    /** public only for testing **/
+    public void incrementRejectionCount() {
         rejectionCount.incrementAndGet();
     }
 
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index e3f16463a5328..e0b35c69cc3c0 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -815,6 +815,7 @@ private static NodeIndicesStats getNodeIndicesStats(boolean remoteStoreStats) {
             remoteSegmentStats.setMaxRefreshTimeLag(2L);
             remoteSegmentStats.addTotalUploadTime(20L);
             remoteSegmentStats.addTotalDownloadTime(20L);
+            remoteSegmentStats.addTotalRejections(5L);
 
             RemoteTranslogStats remoteTranslogStats = indicesStats.getTranslog().getRemoteTranslogStats();
             RemoteTranslogStats otherRemoteTranslogStats = new RemoteTranslogStats(getRandomRemoteTranslogTransferTrackerStats());
diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index b2eb41828a4df..9ef9bec01cb38 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -4910,6 +4910,8 @@ private void populateSampleRemoteSegmentStats(RemoteSegmentTransferTracker track
         tracker.addUploadBytesStarted(30L);
         tracker.addUploadBytesSucceeded(10L);
         tracker.addUploadBytesFailed(10L);
+        tracker.incrementRejectionCount();
+        tracker.incrementRejectionCount();
     }
 
     private void populateSampleRemoteTranslogStats(RemoteTranslogTransferTracker tracker) {
@@ -4943,5 +4945,7 @@ private static void assertRemoteSegmentStats(
         assertEquals(remoteSegmentTransferTracker.getUploadBytesStarted(), remoteSegmentStats.getUploadBytesStarted());
         assertEquals(remoteSegmentTransferTracker.getUploadBytesSucceeded(), remoteSegmentStats.getUploadBytesSucceeded());
         assertEquals(remoteSegmentTransferTracker.getUploadBytesFailed(), remoteSegmentStats.getUploadBytesFailed());
+        assertTrue(remoteSegmentStats.getTotalRejections() > 0);
+        assertEquals(remoteSegmentTransferTracker.getRejectionCount(), remoteSegmentStats.getTotalRejections());
     }
 }

From 01a6ffdf10bfc49341016b244b56bdb1f79e0d11 Mon Sep 17 00:00:00 2001
From: Louis Chu 
Date: Sat, 14 Oct 2023 15:05:37 -0400
Subject: [PATCH 161/388] Update release note and changelog (#10609)

Signed-off-by: Louis Chu 
---
 CHANGELOG.md                                  | 15 ++++++++++++--
 .../opensearch.release-notes-2.11.0.md        | 20 ++++++-------------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 011e404c3d302..644fb05900ceb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,9 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
-- Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
-- Configurable merge policy for index with an option to choose from LogByteSize and Tiered merge policy ([#9992](https://github.com/opensearch-project/OpenSearch/pull/9992))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -88,8 +86,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
 
 ### Dependencies
+- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
+- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
+- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
+- Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))
+- Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
+- Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
+- Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
 
 ### Changed
+- Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
+- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
+- Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
 - [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
 
 ### Deprecated
@@ -97,6 +105,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
+- Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
+- Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
 
 ### Security
 
diff --git a/release-notes/opensearch.release-notes-2.11.0.md b/release-notes/opensearch.release-notes-2.11.0.md
index d7e9182f2a656..7ebf1b433c7c6 100644
--- a/release-notes/opensearch.release-notes-2.11.0.md
+++ b/release-notes/opensearch.release-notes-2.11.0.md
@@ -8,13 +8,16 @@
 - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694))
 - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666))
 - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))
+- Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110))
 - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189))
 - Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562))
 - Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204))
+- Configurable merge policy for index with an option to choose from LogByteSize and Tiered merge policy ([#9992](https://github.com/opensearch-project/OpenSearch/pull/9992))
 - [Remote Store] Add support to restrict creation & deletion if system repository and mutation of immutable settings of system repository ([#9839](https://github.com/opensearch-project/OpenSearch/pull/9839))
 - Improve compressed request handling ([#10261](https://github.com/opensearch-project/OpenSearch/pull/10261))
 
 ### Dependencies
+- Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963))
 - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575))
 - Bump `actions/checkout` from 2 to 4 ([#9968](https://github.com/opensearch-project/OpenSearch/pull/9968))
 - Bump OpenTelemetry from 1.26.0 to 1.30.1 ([#9950](https://github.com/opensearch-project/OpenSearch/pull/9950))
@@ -25,30 +28,22 @@
 - Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098))
 - Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125))
 - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752))
+- Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
+- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
 - Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126))
 - Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.5 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206), [#10299](https://github.com/opensearch-project/OpenSearch/pull/10299))
-- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
-- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
 - Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
 - Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
 - Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
-- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
-- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
 - Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302))
 - Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303))
 - Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301))
 - Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210))
-- Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))
-- Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
-- Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
-- Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
 - Bump `netty` from 4.1.99.Final to 4.1.100.Final ([#10564](https://github.com/opensearch-project/OpenSearch/pull/10564))
 
 ### Changed
 - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
 - Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916))
-- Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
-- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
 - Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042))
 - [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122))
 - Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246))
@@ -58,7 +53,6 @@
 - [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569))
 - [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241))
 - Updating the separator for RemoteStoreLockManager since underscore is allowed in base64UUID url charset ([#10379](https://github.com/opensearch-project/OpenSearch/pull/10379))
-- Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
 
 ### Removed
 - Remove spurious SGID bit on directories ([#9447](https://github.com/opensearch-project/OpenSearch/pull/9447))
@@ -68,8 +62,6 @@
 - Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045))
 - Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082))
 - Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089))
-- Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
-- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194))
 - Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256))
+- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194))
 - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
-- Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))

From ead7ade0b78628defdd28b09ddaf1d276966c619 Mon Sep 17 00:00:00 2001
From: Bhumika Saini 
Date: Mon, 16 Oct 2023 12:00:54 +0530
Subject: [PATCH 162/388] Update version check (#10630)

Signed-off-by: Bhumika Saini 
---
 .../org/opensearch/index/remote/RemoteSegmentStats.java     | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
index 5992923a4157b..4228ec60c4524 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
@@ -95,8 +95,7 @@ public RemoteSegmentStats(StreamInput in) throws IOException {
         totalRefreshBytesLag = in.readLong();
         totalUploadTime = in.readLong();
         totalDownloadTime = in.readLong();
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             totalRejections = in.readVLong();
         }
     }
@@ -260,8 +259,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(totalRefreshBytesLag);
         out.writeLong(totalUploadTime);
         out.writeLong(totalDownloadTime);
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeVLong(totalRejections);
         }
     }

From 368d35abb398b7e379db281358529e4cb689ae05 Mon Sep 17 00:00:00 2001
From: Bharathwaj G 
Date: Mon, 16 Oct 2023 17:01:57 +0530
Subject: [PATCH 163/388] Add resource usage trackers and resource usage
 collector service (#9890)

---------

Signed-off-by: Bharathwaj G 
---
 CHANGELOG.md                                  |   1 +
 .../admin/cluster/node/stats/NodeStats.java   |  23 ++-
 .../cluster/node/stats/NodesStatsRequest.java |   3 +-
 .../node/stats/TransportNodesStatsAction.java |   3 +-
 .../stats/TransportClusterStatsAction.java    |   1 +
 .../common/settings/ClusterSettings.java      |   5 +
 .../main/java/org/opensearch/node/Node.java   |  25 ++-
 .../node/NodeResourceUsageStats.java          |  81 ++++++++
 .../java/org/opensearch/node/NodeService.java |   9 +-
 .../node/NodesResourceUsageStats.java         |  69 +++++++
 .../node/ResourceUsageCollectorService.java   | 160 +++++++++++++++
 .../tracker/AbstractAverageUsageTracker.java  | 100 +++++++++
 .../tracker/AverageCpuUsageTracker.java       |  38 ++++
 .../tracker/AverageMemoryUsageTracker.java    |  42 ++++
 .../tracker/NodeResourceUsageTracker.java     | 118 +++++++++++
 .../tracker/ResourceTrackerSettings.java      |  90 +++++++++
 .../node/resource/tracker/package-info.java   |  12 ++
 .../cluster/node/stats/NodeStatsTests.java    |  45 +++++
 .../opensearch/cluster/DiskUsageTests.java    |   6 +
 .../ResourceUsageCollectorServiceTests.java   | 190 ++++++++++++++++++
 .../tracker/AverageUsageTrackerTests.java     |  99 +++++++++
 .../NodeResourceUsageTrackerTests.java        |  96 +++++++++
 .../MockInternalClusterInfoService.java       |   1 +
 .../opensearch/test/InternalTestCluster.java  |   1 +
 24 files changed, 1211 insertions(+), 7 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/node/NodeResourceUsageStats.java
 create mode 100644 server/src/main/java/org/opensearch/node/NodesResourceUsageStats.java
 create mode 100644 server/src/main/java/org/opensearch/node/ResourceUsageCollectorService.java
 create mode 100644 server/src/main/java/org/opensearch/node/resource/tracker/AbstractAverageUsageTracker.java
 create mode 100644 server/src/main/java/org/opensearch/node/resource/tracker/AverageCpuUsageTracker.java
 create mode 100644 server/src/main/java/org/opensearch/node/resource/tracker/AverageMemoryUsageTracker.java
 create mode 100644 server/src/main/java/org/opensearch/node/resource/tracker/NodeResourceUsageTracker.java
 create mode 100644 server/src/main/java/org/opensearch/node/resource/tracker/ResourceTrackerSettings.java
 create mode 100644 server/src/main/java/org/opensearch/node/resource/tracker/package-info.java
 create mode 100644 server/src/test/java/org/opensearch/node/ResourceUsageCollectorServiceTests.java
 create mode 100644 server/src/test/java/org/opensearch/node/resource/tracker/AverageUsageTrackerTests.java
 create mode 100644 server/src/test/java/org/opensearch/node/resource/tracker/NodeResourceUsageTrackerTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 644fb05900ceb..2211f59574718 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
+- [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index dd36b3b8db3ab..69efea186d927 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -56,6 +56,7 @@
 import org.opensearch.monitor.os.OsStats;
 import org.opensearch.monitor.process.ProcessStats;
 import org.opensearch.node.AdaptiveSelectionStats;
+import org.opensearch.node.NodesResourceUsageStats;
 import org.opensearch.script.ScriptCacheStats;
 import org.opensearch.script.ScriptStats;
 import org.opensearch.search.backpressure.stats.SearchBackpressureStats;
@@ -142,6 +143,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private SearchPipelineStats searchPipelineStats;
 
+    @Nullable
+    private NodesResourceUsageStats resourceUsageStats;
+
     public NodeStats(StreamInput in) throws IOException {
         super(in);
         timestamp = in.readVLong();
@@ -198,6 +202,11 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             searchPipelineStats = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) { // make it 2.12 when we backport
+            resourceUsageStats = in.readOptionalWriteable(NodesResourceUsageStats::new);
+        } else {
+            resourceUsageStats = null;
+        }
     }
 
     public NodeStats(
@@ -216,6 +225,7 @@ public NodeStats(
         @Nullable DiscoveryStats discoveryStats,
         @Nullable IngestStats ingestStats,
         @Nullable AdaptiveSelectionStats adaptiveSelectionStats,
+        @Nullable NodesResourceUsageStats resourceUsageStats,
         @Nullable ScriptCacheStats scriptCacheStats,
         @Nullable IndexingPressureStats indexingPressureStats,
         @Nullable ShardIndexingPressureStats shardIndexingPressureStats,
@@ -241,6 +251,7 @@ public NodeStats(
         this.discoveryStats = discoveryStats;
         this.ingestStats = ingestStats;
         this.adaptiveSelectionStats = adaptiveSelectionStats;
+        this.resourceUsageStats = resourceUsageStats;
         this.scriptCacheStats = scriptCacheStats;
         this.indexingPressureStats = indexingPressureStats;
         this.shardIndexingPressureStats = shardIndexingPressureStats;
@@ -344,6 +355,11 @@ public AdaptiveSelectionStats getAdaptiveSelectionStats() {
         return adaptiveSelectionStats;
     }
 
+    @Nullable
+    public NodesResourceUsageStats getResourceUsageStats() {
+        return resourceUsageStats;
+    }
+
     @Nullable
     public ScriptCacheStats getScriptCacheStats() {
         return scriptCacheStats;
@@ -430,6 +446,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_9_0)) {
             out.writeOptionalWriteable(searchPipelineStats);
         }
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) { // make it 2.12 when we backport
+            out.writeOptionalWriteable(resourceUsageStats);
+        }
     }
 
     @Override
@@ -520,7 +539,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getSearchPipelineStats() != null) {
             getSearchPipelineStats().toXContent(builder, params);
         }
-
+        if (getResourceUsageStats() != null) {
+            getResourceUsageStats().toXContent(builder, params);
+        }
         return builder;
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index b0caa469033eb..99c9fb2d1e26a 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -213,7 +213,8 @@ public enum Metric {
         WEIGHTED_ROUTING_STATS("weighted_routing"),
         FILE_CACHE_STATS("file_cache"),
         TASK_CANCELLATION("task_cancellation"),
-        SEARCH_PIPELINE("search_pipeline");
+        SEARCH_PIPELINE("search_pipeline"),
+        RESOURCE_USAGE_STATS("resource_usage_stats");
 
         private String metricName;
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 615abbaef845d..204157236a282 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -124,7 +124,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics),
             NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics),
             NodesStatsRequest.Metric.TASK_CANCELLATION.containedIn(metrics),
-            NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics)
+            NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics),
+            NodesStatsRequest.Metric.RESOURCE_USAGE_STATS.containedIn(metrics)
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 18098bc31432f..d8323e209be23 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -168,6 +168,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List shardsStats = new ArrayList<>();
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index ad2b89aa3948d..bb0910c4e5f9c 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -130,6 +130,7 @@
 import org.opensearch.node.Node.DiscoverySettings;
 import org.opensearch.node.NodeRoleSettings;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
+import org.opensearch.node.resource.tracker.ResourceTrackerSettings;
 import org.opensearch.persistent.PersistentTasksClusterService;
 import org.opensearch.persistent.decider.EnableAssignmentDecider;
 import org.opensearch.plugins.PluginsService;
@@ -655,6 +656,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SegmentReplicationPressureService.MAX_REPLICATION_LIMIT_STALE_REPLICA_SETTING,
                 SegmentReplicationPressureService.MAX_ALLOWED_STALE_SHARDS,
 
+                // Settings related to resource trackers
+                ResourceTrackerSettings.GLOBAL_CPU_USAGE_AC_WINDOW_DURATION_SETTING,
+                ResourceTrackerSettings.GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING,
+
                 // Settings related to Searchable Snapshots
                 Node.NODE_SEARCH_CACHE_SIZE_SETTING,
                 FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING,
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 5b3b064a47c66..c456f01135dee 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -167,6 +167,7 @@
 import org.opensearch.monitor.fs.FsProbe;
 import org.opensearch.monitor.jvm.JvmInfo;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
+import org.opensearch.node.resource.tracker.NodeResourceUsageTracker;
 import org.opensearch.persistent.PersistentTasksClusterService;
 import org.opensearch.persistent.PersistentTasksExecutor;
 import org.opensearch.persistent.PersistentTasksExecutorRegistry;
@@ -805,7 +806,6 @@ protected Node(
                 remoteStoreStatsTrackerFactory,
                 recoverySettings
             );
-
             final AliasValidator aliasValidator = new AliasValidator();
 
             final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService, systemIndices);
@@ -1070,6 +1070,16 @@ protected Node(
                 transportService.getTaskManager(),
                 taskCancellationMonitoringSettings
             );
+            final NodeResourceUsageTracker nodeResourceUsageTracker = new NodeResourceUsageTracker(
+                threadPool,
+                settings,
+                clusterService.getClusterSettings()
+            );
+            final ResourceUsageCollectorService resourceUsageCollectorService = new ResourceUsageCollectorService(
+                nodeResourceUsageTracker,
+                clusterService,
+                threadPool
+            );
             this.nodeService = new NodeService(
                 settings,
                 threadPool,
@@ -1091,7 +1101,8 @@ protected Node(
                 searchBackpressureService,
                 searchPipelineService,
                 fileCache,
-                taskCancellationMonitoringService
+                taskCancellationMonitoringService,
+                resourceUsageCollectorService
             );
 
             final SearchService searchService = newSearchService(
@@ -1212,6 +1223,8 @@ protected Node(
                 b.bind(RerouteService.class).toInstance(rerouteService);
                 b.bind(ShardLimitValidator.class).toInstance(shardLimitValidator);
                 b.bind(FsHealthService.class).toInstance(fsHealthService);
+                b.bind(NodeResourceUsageTracker.class).toInstance(nodeResourceUsageTracker);
+                b.bind(ResourceUsageCollectorService.class).toInstance(resourceUsageCollectorService);
                 b.bind(SystemIndices.class).toInstance(systemIndices);
                 b.bind(IdentityService.class).toInstance(identityService);
                 b.bind(Tracer.class).toInstance(tracer);
@@ -1328,6 +1341,8 @@ public Node start() throws NodeValidationException {
         injector.getInstance(RepositoriesService.class).start();
         injector.getInstance(SearchService.class).start();
         injector.getInstance(FsHealthService.class).start();
+        injector.getInstance(NodeResourceUsageTracker.class).start();
+        injector.getInstance(ResourceUsageCollectorService.class).start();
         nodeService.getMonitorService().start();
         nodeService.getSearchBackpressureService().start();
         nodeService.getTaskCancellationMonitoringService().start();
@@ -1490,6 +1505,8 @@ private Node stop() {
         injector.getInstance(ClusterService.class).stop();
         injector.getInstance(NodeConnectionsService.class).stop();
         injector.getInstance(FsHealthService.class).stop();
+        injector.getInstance(NodeResourceUsageTracker.class).stop();
+        injector.getInstance(ResourceUsageCollectorService.class).stop();
         nodeService.getMonitorService().stop();
         nodeService.getSearchBackpressureService().stop();
         injector.getInstance(GatewayService.class).stop();
@@ -1553,6 +1570,10 @@ public synchronized void close() throws IOException {
         toClose.add(nodeService.getSearchBackpressureService());
         toClose.add(() -> stopWatch.stop().start("fsHealth"));
         toClose.add(injector.getInstance(FsHealthService.class));
+        toClose.add(() -> stopWatch.stop().start("resource_usage_tracker"));
+        toClose.add(injector.getInstance(NodeResourceUsageTracker.class));
+        toClose.add(() -> stopWatch.stop().start("resource_usage_collector"));
+        toClose.add(injector.getInstance(ResourceUsageCollectorService.class));
         toClose.add(() -> stopWatch.stop().start("gateway"));
         toClose.add(injector.getInstance(GatewayService.class));
         toClose.add(() -> stopWatch.stop().start("search"));
diff --git a/server/src/main/java/org/opensearch/node/NodeResourceUsageStats.java b/server/src/main/java/org/opensearch/node/NodeResourceUsageStats.java
new file mode 100644
index 0000000000000..6ef66d4ac1914
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/NodeResourceUsageStats.java
@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/**
+ * This represents the resource usage stats of a node along with the timestamp at which the stats object was created
+ * in the respective node
+ */
+public class NodeResourceUsageStats implements Writeable {
+    final String nodeId;
+    long timestamp;
+    double cpuUtilizationPercent;
+    double memoryUtilizationPercent;
+
+    public NodeResourceUsageStats(String nodeId, long timestamp, double memoryUtilizationPercent, double cpuUtilizationPercent) {
+        this.nodeId = nodeId;
+        this.timestamp = timestamp;
+        this.cpuUtilizationPercent = cpuUtilizationPercent;
+        this.memoryUtilizationPercent = memoryUtilizationPercent;
+    }
+
+    public NodeResourceUsageStats(StreamInput in) throws IOException {
+        this.nodeId = in.readString();
+        this.timestamp = in.readLong();
+        this.cpuUtilizationPercent = in.readDouble();
+        this.memoryUtilizationPercent = in.readDouble();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(this.nodeId);
+        out.writeLong(this.timestamp);
+        out.writeDouble(this.cpuUtilizationPercent);
+        out.writeDouble(this.memoryUtilizationPercent);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder("NodeResourceUsageStats[");
+        sb.append(nodeId).append("](");
+        sb.append("Timestamp: ").append(timestamp);
+        sb.append(", CPU utilization percent: ").append(String.format(Locale.ROOT, "%.1f", cpuUtilizationPercent));
+        sb.append(", Memory utilization percent: ").append(String.format(Locale.ROOT, "%.1f", memoryUtilizationPercent));
+        sb.append(")");
+        return sb.toString();
+    }
+
+    NodeResourceUsageStats(NodeResourceUsageStats nodeResourceUsageStats) {
+        this(
+            nodeResourceUsageStats.nodeId,
+            nodeResourceUsageStats.timestamp,
+            nodeResourceUsageStats.memoryUtilizationPercent,
+            nodeResourceUsageStats.cpuUtilizationPercent
+        );
+    }
+
+    public double getMemoryUtilizationPercent() {
+        return memoryUtilizationPercent;
+    }
+
+    public double getCpuUtilizationPercent() {
+        return cpuUtilizationPercent;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java
index 2688b894cb9a7..9bb07080fa717 100644
--- a/server/src/main/java/org/opensearch/node/NodeService.java
+++ b/server/src/main/java/org/opensearch/node/NodeService.java
@@ -83,6 +83,7 @@ public class NodeService implements Closeable {
     private final ScriptService scriptService;
     private final HttpServerTransport httpServerTransport;
     private final ResponseCollectorService responseCollectorService;
+    private final ResourceUsageCollectorService resourceUsageCollectorService;
     private final SearchTransportService searchTransportService;
     private final IndexingPressureService indexingPressureService;
     private final AggregationUsageService aggregationUsageService;
@@ -114,7 +115,8 @@ public class NodeService implements Closeable {
         SearchBackpressureService searchBackpressureService,
         SearchPipelineService searchPipelineService,
         FileCache fileCache,
-        TaskCancellationMonitoringService taskCancellationMonitoringService
+        TaskCancellationMonitoringService taskCancellationMonitoringService,
+        ResourceUsageCollectorService resourceUsageCollectorService
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -137,6 +139,7 @@ public class NodeService implements Closeable {
         this.clusterService = clusterService;
         this.fileCache = fileCache;
         this.taskCancellationMonitoringService = taskCancellationMonitoringService;
+        this.resourceUsageCollectorService = resourceUsageCollectorService;
         clusterService.addStateApplier(ingestService);
         clusterService.addStateApplier(searchPipelineService);
     }
@@ -217,7 +220,8 @@ public NodeStats stats(
         boolean weightedRoutingStats,
         boolean fileCacheStats,
         boolean taskCancellation,
-        boolean searchPipelineStats
+        boolean searchPipelineStats,
+        boolean resourceUsageStats
     ) {
         // for indices stats we want to include previous allocated shards stats as well (it will
         // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
@@ -237,6 +241,7 @@ public NodeStats stats(
             discoveryStats ? discovery.stats() : null,
             ingest ? ingestService.stats() : null,
             adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getPendingSearchRequests()) : null,
+            resourceUsageStats ? resourceUsageCollectorService.stats() : null,
             scriptCache ? scriptService.cacheStats() : null,
             indexingPressure ? this.indexingPressureService.nodeStats() : null,
             shardIndexingPressure ? this.indexingPressureService.shardStats(indices) : null,
diff --git a/server/src/main/java/org/opensearch/node/NodesResourceUsageStats.java b/server/src/main/java/org/opensearch/node/NodesResourceUsageStats.java
new file mode 100644
index 0000000000000..3dff9a27f71a8
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/NodesResourceUsageStats.java
@@ -0,0 +1,69 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * This class represents resource usage stats such as CPU, Memory and IO resource usage of each node along with the
+ * timestamp of the stats recorded.
+ */
+public class NodesResourceUsageStats implements Writeable, ToXContentFragment {
+
+    // Map of node id to resource usage stats of the corresponding node.
+    private final Map nodeIdToResourceUsageStatsMap;
+
+    public NodesResourceUsageStats(Map nodeIdToResourceUsageStatsMap) {
+        this.nodeIdToResourceUsageStatsMap = nodeIdToResourceUsageStatsMap;
+    }
+
+    public NodesResourceUsageStats(StreamInput in) throws IOException {
+        this.nodeIdToResourceUsageStatsMap = in.readMap(StreamInput::readString, NodeResourceUsageStats::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(this.nodeIdToResourceUsageStatsMap, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream));
+    }
+
+    /**
+     * Returns map of node id to resource usage stats of the corresponding node.
+     */
+    public Map getNodeIdToResourceUsageStatsMap() {
+        return nodeIdToResourceUsageStatsMap;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject("resource_usage_stats");
+        for (String nodeId : nodeIdToResourceUsageStatsMap.keySet()) {
+            builder.startObject(nodeId);
+            NodeResourceUsageStats resourceUsageStats = nodeIdToResourceUsageStatsMap.get(nodeId);
+            if (resourceUsageStats != null) {
+                builder.field("timestamp", resourceUsageStats.timestamp);
+                builder.field("cpu_utilization_percent", String.format(Locale.ROOT, "%.1f", resourceUsageStats.cpuUtilizationPercent));
+                builder.field(
+                    "memory_utilization_percent",
+                    String.format(Locale.ROOT, "%.1f", resourceUsageStats.memoryUtilizationPercent)
+                );
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/node/ResourceUsageCollectorService.java b/server/src/main/java/org/opensearch/node/ResourceUsageCollectorService.java
new file mode 100644
index 0000000000000..f1c763e09f147
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/ResourceUsageCollectorService.java
@@ -0,0 +1,160 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterStateListener;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.concurrent.ConcurrentCollections;
+import org.opensearch.node.resource.tracker.NodeResourceUsageTracker;
+import org.opensearch.threadpool.Scheduler;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * This collects node level resource usage statistics such as cpu, memory, IO of each node and makes it available for
+ * coordinator node to aid in throttling, ranking etc
+ */
+public class ResourceUsageCollectorService extends AbstractLifecycleComponent implements ClusterStateListener {
+
+    /**
+     * This refresh interval denotes the polling interval of ResourceUsageCollectorService to refresh the resource usage
+     * stats from local node
+     */
+    private static long REFRESH_INTERVAL_IN_MILLIS = 1000;
+
+    private static final Logger logger = LogManager.getLogger(ResourceUsageCollectorService.class);
+    private final ConcurrentMap nodeIdToResourceUsageStats = ConcurrentCollections.newConcurrentMap();
+
+    private ThreadPool threadPool;
+    private volatile Scheduler.Cancellable scheduledFuture;
+
+    private NodeResourceUsageTracker nodeResourceUsageTracker;
+    private ClusterService clusterService;
+
+    public ResourceUsageCollectorService(
+        NodeResourceUsageTracker nodeResourceUsageTracker,
+        ClusterService clusterService,
+        ThreadPool threadPool
+    ) {
+        this.threadPool = threadPool;
+        this.nodeResourceUsageTracker = nodeResourceUsageTracker;
+        this.clusterService = clusterService;
+        clusterService.addListener(this);
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        if (event.nodesRemoved()) {
+            for (DiscoveryNode removedNode : event.nodesDelta().removedNodes()) {
+                removeNodeResourceUsageStats(removedNode.getId());
+            }
+        }
+    }
+
+    void removeNodeResourceUsageStats(String nodeId) {
+        nodeIdToResourceUsageStats.remove(nodeId);
+    }
+
+    /**
+     * Collect node resource usage stats along with the timestamp
+     */
+    public void collectNodeResourceUsageStats(
+        String nodeId,
+        long timestamp,
+        double memoryUtilizationPercent,
+        double cpuUtilizationPercent
+    ) {
+        nodeIdToResourceUsageStats.compute(nodeId, (id, resourceUsageStats) -> {
+            if (resourceUsageStats == null) {
+                return new NodeResourceUsageStats(nodeId, timestamp, memoryUtilizationPercent, cpuUtilizationPercent);
+            } else {
+                resourceUsageStats.cpuUtilizationPercent = cpuUtilizationPercent;
+                resourceUsageStats.memoryUtilizationPercent = memoryUtilizationPercent;
+                resourceUsageStats.timestamp = timestamp;
+                return resourceUsageStats;
+            }
+        });
+    }
+
+    /**
+     * Get all node resource usage statistics which will be used for node stats
+     */
+    public Map getAllNodeStatistics() {
+        Map nodeStats = new HashMap<>(nodeIdToResourceUsageStats.size());
+        nodeIdToResourceUsageStats.forEach((nodeId, resourceUsageStats) -> {
+            nodeStats.put(nodeId, new NodeResourceUsageStats(resourceUsageStats));
+        });
+        return nodeStats;
+    }
+
+    /**
+     * Optionally return a {@code NodeResourceUsageStats} for the given nodeid, if
+     * resource usage stats information exists for the given node. Returns an empty
+     * {@code Optional} if the node was not found.
+     */
+    public Optional getNodeStatistics(final String nodeId) {
+        return Optional.ofNullable(nodeIdToResourceUsageStats.get(nodeId))
+            .map(resourceUsageStats -> new NodeResourceUsageStats(resourceUsageStats));
+    }
+
+    /**
+     * Returns collected resource usage statistics of all nodes
+     */
+    public NodesResourceUsageStats stats() {
+        return new NodesResourceUsageStats(getAllNodeStatistics());
+    }
+
+    /**
+     * Fetch local node resource usage statistics and add it to store along with the current timestamp
+     */
+    private void collectLocalNodeResourceUsageStats() {
+        if (nodeResourceUsageTracker.isReady() && clusterService.state() != null) {
+            collectNodeResourceUsageStats(
+                clusterService.state().nodes().getLocalNodeId(),
+                System.currentTimeMillis(),
+                nodeResourceUsageTracker.getMemoryUtilizationPercent(),
+                nodeResourceUsageTracker.getCpuUtilizationPercent()
+            );
+        }
+    }
+
+    @Override
+    protected void doStart() {
+        /**
+         * Fetch local node resource usage statistics every second
+         */
+        scheduledFuture = threadPool.scheduleWithFixedDelay(() -> {
+            try {
+                collectLocalNodeResourceUsageStats();
+            } catch (Exception e) {
+                logger.warn("failure in ResourceUsageCollectorService", e);
+            }
+        }, new TimeValue(REFRESH_INTERVAL_IN_MILLIS), ThreadPool.Names.GENERIC);
+    }
+
+    @Override
+    protected void doStop() {
+        if (scheduledFuture != null) {
+            scheduledFuture.cancel();
+        }
+    }
+
+    @Override
+    protected void doClose() {}
+}
diff --git a/server/src/main/java/org/opensearch/node/resource/tracker/AbstractAverageUsageTracker.java b/server/src/main/java/org/opensearch/node/resource/tracker/AbstractAverageUsageTracker.java
new file mode 100644
index 0000000000000..f83a1b7f9fc05
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/resource/tracker/AbstractAverageUsageTracker.java
@@ -0,0 +1,100 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.resource.tracker;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.MovingAverage;
+import org.opensearch.threadpool.Scheduler;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Base class for sliding window resource usage trackers
+ */
+public abstract class AbstractAverageUsageTracker extends AbstractLifecycleComponent {
+    private static final Logger LOGGER = LogManager.getLogger(AbstractAverageUsageTracker.class);
+
+    private final ThreadPool threadPool;
+    private final TimeValue pollingInterval;
+    private TimeValue windowDuration;
+    private final AtomicReference observations = new AtomicReference<>();
+
+    private volatile Scheduler.Cancellable scheduledFuture;
+
+    public AbstractAverageUsageTracker(ThreadPool threadPool, TimeValue pollingInterval, TimeValue windowDuration) {
+        this.threadPool = threadPool;
+        this.pollingInterval = pollingInterval;
+        this.windowDuration = windowDuration;
+        this.setWindowSize(windowDuration);
+    }
+
+    public abstract long getUsage();
+
+    /**
+     * Returns the moving average of the datapoints
+     */
+    public double getAverage() {
+        return observations.get().getAverage();
+    }
+
+    /**
+     * Checks if we have datapoints more than or equal to the window size
+     */
+    public boolean isReady() {
+        return observations.get().isReady();
+    }
+
+    /**
+     * Creates a new instance of MovingAverage with a new window size based on WindowDuration
+     */
+    public void setWindowSize(TimeValue windowDuration) {
+        this.windowDuration = windowDuration;
+        int windowSize = (int) (windowDuration.nanos() / pollingInterval.nanos());
+        LOGGER.debug("updated window size: {}", windowSize);
+        observations.set(new MovingAverage(windowSize));
+    }
+
+    public TimeValue getPollingInterval() {
+        return pollingInterval;
+    }
+
+    public TimeValue getWindowDuration() {
+        return windowDuration;
+    }
+
+    public long getWindowSize() {
+        return observations.get().getCount();
+    }
+
+    public void recordUsage(long usage) {
+        observations.get().record(usage);
+    }
+
+    @Override
+    protected void doStart() {
+        scheduledFuture = threadPool.scheduleWithFixedDelay(() -> {
+            long usage = getUsage();
+            recordUsage(usage);
+        }, pollingInterval, ThreadPool.Names.GENERIC);
+    }
+
+    @Override
+    protected void doStop() {
+        if (scheduledFuture != null) {
+            scheduledFuture.cancel();
+        }
+    }
+
+    @Override
+    protected void doClose() {}
+}
diff --git a/server/src/main/java/org/opensearch/node/resource/tracker/AverageCpuUsageTracker.java b/server/src/main/java/org/opensearch/node/resource/tracker/AverageCpuUsageTracker.java
new file mode 100644
index 0000000000000..160d385762eb0
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/resource/tracker/AverageCpuUsageTracker.java
@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.resource.tracker;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.monitor.process.ProcessProbe;
+import org.opensearch.threadpool.ThreadPool;
+
+/**
+ * AverageCpuUsageTracker tracks the average CPU usage by polling the CPU usage every (pollingInterval)
+ * and keeping track of the rolling average over a defined time window (windowDuration).
+ */
+public class AverageCpuUsageTracker extends AbstractAverageUsageTracker {
+    private static final Logger LOGGER = LogManager.getLogger(AverageCpuUsageTracker.class);
+
+    public AverageCpuUsageTracker(ThreadPool threadPool, TimeValue pollingInterval, TimeValue windowDuration) {
+        super(threadPool, pollingInterval, windowDuration);
+    }
+
+    /**
+     * Returns the process CPU usage in percent
+     */
+    @Override
+    public long getUsage() {
+        long usage = ProcessProbe.getInstance().getProcessCpuPercent();
+        LOGGER.debug("Recording cpu usage: {}%", usage);
+        return usage;
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/node/resource/tracker/AverageMemoryUsageTracker.java b/server/src/main/java/org/opensearch/node/resource/tracker/AverageMemoryUsageTracker.java
new file mode 100644
index 0000000000000..c1d1c83656859
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/resource/tracker/AverageMemoryUsageTracker.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.resource.tracker;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+
+/**
+ * AverageMemoryUsageTracker tracks the average JVM usage by polling the JVM usage every (pollingInterval)
+ * and keeping track of the rolling average over a defined time window (windowDuration).
+ */
+public class AverageMemoryUsageTracker extends AbstractAverageUsageTracker {
+
+    private static final Logger LOGGER = LogManager.getLogger(AverageMemoryUsageTracker.class);
+
+    private static final MemoryMXBean MEMORY_MX_BEAN = ManagementFactory.getMemoryMXBean();
+
+    public AverageMemoryUsageTracker(ThreadPool threadPool, TimeValue pollingInterval, TimeValue windowDuration) {
+        super(threadPool, pollingInterval, windowDuration);
+    }
+
+    /**
+     * Get current memory usage percentage calculated against max heap memory
+     */
+    @Override
+    public long getUsage() {
+        long usage = MEMORY_MX_BEAN.getHeapMemoryUsage().getUsed() * 100 / MEMORY_MX_BEAN.getHeapMemoryUsage().getMax();
+        LOGGER.debug("Recording memory usage: {}%", usage);
+        return usage;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/node/resource/tracker/NodeResourceUsageTracker.java b/server/src/main/java/org/opensearch/node/resource/tracker/NodeResourceUsageTracker.java
new file mode 100644
index 0000000000000..cf5f38c1b004c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/resource/tracker/NodeResourceUsageTracker.java
@@ -0,0 +1,118 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.resource.tracker;
+
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.threadpool.ThreadPool;
+
+/**
+ * This tracks the usage of node resources such as CPU, IO and memory
+ */
+public class NodeResourceUsageTracker extends AbstractLifecycleComponent {
+    private ThreadPool threadPool;
+    private final ClusterSettings clusterSettings;
+    private AverageCpuUsageTracker cpuUsageTracker;
+    private AverageMemoryUsageTracker memoryUsageTracker;
+
+    private ResourceTrackerSettings resourceTrackerSettings;
+
+    public NodeResourceUsageTracker(ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings) {
+        this.threadPool = threadPool;
+        this.clusterSettings = clusterSettings;
+        this.resourceTrackerSettings = new ResourceTrackerSettings(settings);
+        initialize();
+    }
+
+    /**
+     * Return CPU utilization average if we have enough datapoints, otherwise return 0
+     */
+    public double getCpuUtilizationPercent() {
+        if (cpuUsageTracker.isReady()) {
+            return cpuUsageTracker.getAverage();
+        }
+        return 0.0;
+    }
+
+    /**
+     * Return memory utilization average if we have enough datapoints, otherwise return 0
+     */
+    public double getMemoryUtilizationPercent() {
+        if (memoryUsageTracker.isReady()) {
+            return memoryUsageTracker.getAverage();
+        }
+        return 0.0;
+    }
+
+    /**
+     * Checks if all of the resource usage trackers are ready
+     */
+    public boolean isReady() {
+        return memoryUsageTracker.isReady() && cpuUsageTracker.isReady();
+    }
+
+    void initialize() {
+        cpuUsageTracker = new AverageCpuUsageTracker(
+            threadPool,
+            resourceTrackerSettings.getCpuPollingInterval(),
+            resourceTrackerSettings.getCpuWindowDuration()
+        );
+        clusterSettings.addSettingsUpdateConsumer(
+            ResourceTrackerSettings.GLOBAL_CPU_USAGE_AC_WINDOW_DURATION_SETTING,
+            this::setCpuWindowDuration
+        );
+
+        memoryUsageTracker = new AverageMemoryUsageTracker(
+            threadPool,
+            resourceTrackerSettings.getMemoryPollingInterval(),
+            resourceTrackerSettings.getMemoryWindowDuration()
+        );
+        clusterSettings.addSettingsUpdateConsumer(
+            ResourceTrackerSettings.GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING,
+            this::setMemoryWindowDuration
+        );
+    }
+
+    private void setMemoryWindowDuration(TimeValue windowDuration) {
+        memoryUsageTracker.setWindowSize(windowDuration);
+        resourceTrackerSettings.setMemoryWindowDuration(windowDuration);
+    }
+
+    private void setCpuWindowDuration(TimeValue windowDuration) {
+        cpuUsageTracker.setWindowSize(windowDuration);
+        resourceTrackerSettings.setCpuWindowDuration(windowDuration);
+    }
+
+    /**
+     * Visible for testing
+     */
+    ResourceTrackerSettings getResourceTrackerSettings() {
+        return resourceTrackerSettings;
+    }
+
+    @Override
+    protected void doStart() {
+        cpuUsageTracker.doStart();
+        memoryUsageTracker.doStart();
+    }
+
+    @Override
+    protected void doStop() {
+        cpuUsageTracker.doStop();
+        memoryUsageTracker.doStop();
+    }
+
+    @Override
+    protected void doClose() {
+        cpuUsageTracker.doClose();
+        memoryUsageTracker.doClose();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/node/resource/tracker/ResourceTrackerSettings.java b/server/src/main/java/org/opensearch/node/resource/tracker/ResourceTrackerSettings.java
new file mode 100644
index 0000000000000..f81b008ba7e8b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/resource/tracker/ResourceTrackerSettings.java
@@ -0,0 +1,90 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.resource.tracker;
+
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+
+/**
+ * Settings related to resource usage trackers such as polling interval, window duration etc
+ */
+public class ResourceTrackerSettings {
+
+    private static class Defaults {
+        /**
+         * This is the default polling interval of usage trackers to get the resource utilization data
+         */
+        private static final long POLLING_INTERVAL_IN_MILLIS = 500;
+        /**
+         * This is the default window duration on which the average resource utilization values will be calculated
+         */
+        private static final long WINDOW_DURATION_IN_SECONDS = 30;
+    }
+
+    public static final Setting GLOBAL_CPU_USAGE_AC_POLLING_INTERVAL_SETTING = Setting.positiveTimeSetting(
+        "node.resource.tracker.global_cpu_usage.polling_interval",
+        TimeValue.timeValueMillis(Defaults.POLLING_INTERVAL_IN_MILLIS),
+        Setting.Property.NodeScope
+    );
+    public static final Setting GLOBAL_CPU_USAGE_AC_WINDOW_DURATION_SETTING = Setting.positiveTimeSetting(
+        "node.resource.tracker.global_cpu_usage.window_duration",
+        TimeValue.timeValueSeconds(Defaults.WINDOW_DURATION_IN_SECONDS),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting GLOBAL_JVM_USAGE_AC_POLLING_INTERVAL_SETTING = Setting.positiveTimeSetting(
+        "node.resource.tracker.global_jvmmp.polling_interval",
+        TimeValue.timeValueMillis(Defaults.POLLING_INTERVAL_IN_MILLIS),
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING = Setting.positiveTimeSetting(
+        "node.resource.tracker.global_jvmmp.window_duration",
+        TimeValue.timeValueSeconds(Defaults.WINDOW_DURATION_IN_SECONDS),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+    private volatile TimeValue cpuWindowDuration;
+    private volatile TimeValue cpuPollingInterval;
+    private volatile TimeValue memoryWindowDuration;
+    private volatile TimeValue memoryPollingInterval;
+
+    public ResourceTrackerSettings(Settings settings) {
+        this.cpuPollingInterval = GLOBAL_CPU_USAGE_AC_POLLING_INTERVAL_SETTING.get(settings);
+        this.cpuWindowDuration = GLOBAL_CPU_USAGE_AC_WINDOW_DURATION_SETTING.get(settings);
+        this.memoryPollingInterval = GLOBAL_JVM_USAGE_AC_POLLING_INTERVAL_SETTING.get(settings);
+        this.memoryWindowDuration = GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING.get(settings);
+    }
+
+    public TimeValue getCpuWindowDuration() {
+        return this.cpuWindowDuration;
+    }
+
+    public TimeValue getCpuPollingInterval() {
+        return cpuPollingInterval;
+    }
+
+    public TimeValue getMemoryPollingInterval() {
+        return memoryPollingInterval;
+    }
+
+    public TimeValue getMemoryWindowDuration() {
+        return memoryWindowDuration;
+    }
+
+    public void setCpuWindowDuration(TimeValue cpuWindowDuration) {
+        this.cpuWindowDuration = cpuWindowDuration;
+    }
+
+    public void setMemoryWindowDuration(TimeValue memoryWindowDuration) {
+        this.memoryWindowDuration = memoryWindowDuration;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/node/resource/tracker/package-info.java b/server/src/main/java/org/opensearch/node/resource/tracker/package-info.java
new file mode 100644
index 0000000000000..aace2a019973e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/node/resource/tracker/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Node level resource usage stats tracker package
+ */
+package org.opensearch.node.resource.tracker;
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index e0b35c69cc3c0..7a1b6f113d0e8 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -59,6 +59,8 @@
 import org.opensearch.monitor.os.OsStats;
 import org.opensearch.monitor.process.ProcessStats;
 import org.opensearch.node.AdaptiveSelectionStats;
+import org.opensearch.node.NodeResourceUsageStats;
+import org.opensearch.node.NodesResourceUsageStats;
 import org.opensearch.node.ResponseCollectorService;
 import org.opensearch.script.ScriptCacheStats;
 import org.opensearch.script.ScriptStats;
@@ -393,6 +395,24 @@ public void testSerialization() throws IOException {
                         assertEquals(aStats.responseTime, bStats.responseTime, 0.01);
                     });
                 }
+                NodesResourceUsageStats resourceUsageStats = nodeStats.getResourceUsageStats();
+                NodesResourceUsageStats deserializedResourceUsageStats = deserializedNodeStats.getResourceUsageStats();
+                if (resourceUsageStats == null) {
+                    assertNull(deserializedResourceUsageStats);
+                } else {
+                    resourceUsageStats.getNodeIdToResourceUsageStatsMap().forEach((k, v) -> {
+                        NodeResourceUsageStats aResourceUsageStats = resourceUsageStats.getNodeIdToResourceUsageStatsMap().get(k);
+                        NodeResourceUsageStats bResourceUsageStats = deserializedResourceUsageStats.getNodeIdToResourceUsageStatsMap()
+                            .get(k);
+                        assertEquals(
+                            aResourceUsageStats.getMemoryUtilizationPercent(),
+                            bResourceUsageStats.getMemoryUtilizationPercent(),
+                            0.0
+                        );
+                        assertEquals(aResourceUsageStats.getCpuUtilizationPercent(), bResourceUsageStats.getCpuUtilizationPercent(), 0.0);
+                        assertEquals(aResourceUsageStats.getTimestamp(), bResourceUsageStats.getTimestamp());
+                    });
+                }
                 ScriptCacheStats scriptCacheStats = nodeStats.getScriptCacheStats();
                 ScriptCacheStats deserializedScriptCacheStats = deserializedNodeStats.getScriptCacheStats();
                 if (scriptCacheStats == null) {
@@ -756,6 +776,30 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
             }
             adaptiveSelectionStats = new AdaptiveSelectionStats(nodeConnections, nodeStats);
         }
+        NodesResourceUsageStats nodesResourceUsageStats = null;
+        if (frequently()) {
+            int numNodes = randomIntBetween(0, 10);
+            Map nodeConnections = new HashMap<>();
+            Map resourceUsageStatsMap = new HashMap<>();
+            for (int i = 0; i < numNodes; i++) {
+                String nodeId = randomAlphaOfLengthBetween(3, 10);
+                // add outgoing connection info
+                if (frequently()) {
+                    nodeConnections.put(nodeId, randomLongBetween(0, 100));
+                }
+                // add node calculations
+                if (frequently()) {
+                    NodeResourceUsageStats stats = new NodeResourceUsageStats(
+                        nodeId,
+                        System.currentTimeMillis(),
+                        randomDoubleBetween(1.0, 100.0, true),
+                        randomDoubleBetween(1.0, 100.0, true)
+                    );
+                    resourceUsageStatsMap.put(nodeId, stats);
+                }
+            }
+            nodesResourceUsageStats = new NodesResourceUsageStats(resourceUsageStatsMap);
+        }
         ClusterManagerThrottlingStats clusterManagerThrottlingStats = null;
         if (frequently()) {
             clusterManagerThrottlingStats = new ClusterManagerThrottlingStats();
@@ -787,6 +831,7 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
             discoveryStats,
             ingestStats,
             adaptiveSelectionStats,
+            nodesResourceUsageStats,
             scriptCacheStats,
             null,
             null,
diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
index 8ba965b3df1ab..64949cf861f70 100644
--- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
@@ -190,6 +190,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -216,6 +217,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -242,6 +244,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
@@ -299,6 +302,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -325,6 +329,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -351,6 +356,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
diff --git a/server/src/test/java/org/opensearch/node/ResourceUsageCollectorServiceTests.java b/server/src/test/java/org/opensearch/node/ResourceUsageCollectorServiceTests.java
new file mode 100644
index 0000000000000..b2fa884afab69
--- /dev/null
+++ b/server/src/test/java/org/opensearch/node/ResourceUsageCollectorServiceTests.java
@@ -0,0 +1,190 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node;
+
+import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterName;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.node.resource.tracker.NodeResourceUsageTracker;
+import org.opensearch.node.resource.tracker.ResourceTrackerSettings;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.opensearch.test.ClusterServiceUtils.createClusterService;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * Tests for ResourceUsageCollectorService where we test collect method, get method and whether schedulers
+ * are working as expected
+ */
+public class ResourceUsageCollectorServiceTests extends OpenSearchSingleNodeTestCase {
+
+    private ClusterService clusterService;
+    private ResourceUsageCollectorService collector;
+    private ThreadPool threadpool;
+    NodeResourceUsageTracker tracker;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+
+        threadpool = new TestThreadPool("resource_usage_collector_tests");
+
+        clusterService = createClusterService(threadpool);
+
+        Settings settings = Settings.builder()
+            .put(ResourceTrackerSettings.GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING.getKey(), new TimeValue(500, TimeUnit.MILLISECONDS))
+            .build();
+        tracker = new NodeResourceUsageTracker(
+            threadpool,
+            settings,
+            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        collector = new ResourceUsageCollectorService(tracker, clusterService, threadpool);
+        tracker.start();
+        collector.start();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadpool.shutdownNow();
+        clusterService.close();
+        collector.stop();
+        tracker.stop();
+        collector.close();
+        tracker.close();
+    }
+
+    public void testResourceUsageStats() {
+        collector.collectNodeResourceUsageStats("node1", System.currentTimeMillis(), 97, 99);
+        Map nodeStats = collector.getAllNodeStatistics();
+        assertTrue(nodeStats.containsKey("node1"));
+        assertEquals(99.0, nodeStats.get("node1").cpuUtilizationPercent, 0.0);
+        assertEquals(97.0, nodeStats.get("node1").memoryUtilizationPercent, 0.0);
+
+        Optional nodeResourceUsageStatsOptional = collector.getNodeStatistics("node1");
+
+        assertNotNull(nodeResourceUsageStatsOptional.get());
+        assertEquals(99.0, nodeResourceUsageStatsOptional.get().cpuUtilizationPercent, 0.0);
+        assertEquals(97.0, nodeResourceUsageStatsOptional.get().memoryUtilizationPercent, 0.0);
+
+        nodeResourceUsageStatsOptional = collector.getNodeStatistics("node2");
+        assertTrue(nodeResourceUsageStatsOptional.isEmpty());
+    }
+
+    public void testScheduler() throws Exception {
+        /**
+         * Wait for cluster state to be ready so that localNode().getId() is ready and we add the values to the map
+         */
+        assertBusy(() -> assertTrue(collector.getNodeStatistics(clusterService.localNode().getId()).isPresent()), 1, TimeUnit.MINUTES);
+        assertTrue(collector.getNodeStatistics(clusterService.localNode().getId()).isPresent());
+        /**
+         * Wait for memory utilization to be reported greater than 0
+         */
+        assertBusy(
+            () -> assertThat(
+                collector.getNodeStatistics(clusterService.localNode().getId()).get().getMemoryUtilizationPercent(),
+                greaterThan(0.0)
+            ),
+            5,
+            TimeUnit.SECONDS
+        );
+        assertTrue(collector.getNodeStatistics("Invalid").isEmpty());
+    }
+
+    /*
+     * Test that concurrently adding values and removing nodes does not cause exceptions
+     */
+    public void testConcurrentAddingAndRemovingNodes() throws Exception {
+        String[] nodes = new String[] { "a", "b", "c", "d" };
+
+        final CountDownLatch latch = new CountDownLatch(5);
+
+        Runnable f = () -> {
+            latch.countDown();
+            try {
+                latch.await();
+            } catch (InterruptedException e) {
+                fail("should not be interrupted");
+            }
+            for (int i = 0; i < randomIntBetween(100, 200); i++) {
+                if (randomBoolean()) {
+                    collector.removeNodeResourceUsageStats(randomFrom(nodes));
+                }
+                collector.collectNodeResourceUsageStats(
+                    randomFrom(nodes),
+                    System.currentTimeMillis(),
+                    randomIntBetween(1, 100),
+                    randomIntBetween(1, 100)
+                );
+            }
+        };
+
+        Thread t1 = new Thread(f);
+        Thread t2 = new Thread(f);
+        Thread t3 = new Thread(f);
+        Thread t4 = new Thread(f);
+
+        t1.start();
+        t2.start();
+        t3.start();
+        t4.start();
+        latch.countDown();
+        t1.join();
+        t2.join();
+        t3.join();
+        t4.join();
+
+        final Map nodeStats = collector.getAllNodeStatistics();
+        for (String nodeId : nodes) {
+            if (nodeStats.containsKey(nodeId)) {
+                assertThat(nodeStats.get(nodeId).memoryUtilizationPercent, greaterThan(0.0));
+                assertThat(nodeStats.get(nodeId).cpuUtilizationPercent, greaterThan(0.0));
+            }
+        }
+    }
+
+    public void testNodeRemoval() {
+        collector.collectNodeResourceUsageStats("node1", System.currentTimeMillis(), randomIntBetween(1, 100), randomIntBetween(1, 100));
+        collector.collectNodeResourceUsageStats("node2", System.currentTimeMillis(), randomIntBetween(1, 100), randomIntBetween(1, 100));
+
+        ClusterState previousState = ClusterState.builder(new ClusterName("cluster"))
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9200), "node1"))
+                    .add(DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9201), "node2"))
+            )
+            .build();
+        ClusterState newState = ClusterState.builder(previousState)
+            .nodes(DiscoveryNodes.builder(previousState.nodes()).remove("node2"))
+            .build();
+        ClusterChangedEvent event = new ClusterChangedEvent("test", newState, previousState);
+
+        collector.clusterChanged(event);
+        final Map nodeStats = collector.getAllNodeStatistics();
+        assertTrue(nodeStats.containsKey("node1"));
+        assertFalse(nodeStats.containsKey("node2"));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/node/resource/tracker/AverageUsageTrackerTests.java b/server/src/test/java/org/opensearch/node/resource/tracker/AverageUsageTrackerTests.java
new file mode 100644
index 0000000000000..374c993a264d4
--- /dev/null
+++ b/server/src/test/java/org/opensearch/node/resource/tracker/AverageUsageTrackerTests.java
@@ -0,0 +1,99 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.resource.tracker;
+
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests to validate AverageMemoryUsageTracker and AverageCpuUsageTracker implementation
+ */
+public class AverageUsageTrackerTests extends OpenSearchTestCase {
+    ThreadPool threadPool;
+    AverageMemoryUsageTracker averageMemoryUsageTracker;
+    AverageCpuUsageTracker averageCpuUsageTracker;
+
+    @Before
+    public void setup() {
+        threadPool = new TestThreadPool(getClass().getName());
+        averageMemoryUsageTracker = new AverageMemoryUsageTracker(
+            threadPool,
+            new TimeValue(500, TimeUnit.MILLISECONDS),
+            new TimeValue(1000, TimeUnit.MILLISECONDS)
+        );
+        averageCpuUsageTracker = new AverageCpuUsageTracker(
+            threadPool,
+            new TimeValue(500, TimeUnit.MILLISECONDS),
+            new TimeValue(1000, TimeUnit.MILLISECONDS)
+        );
+    }
+
+    @After
+    public void cleanup() {
+        ThreadPool.terminate(threadPool, 5, TimeUnit.SECONDS);
+    }
+
+    public void testBasicUsage() {
+
+        assertAverageUsageStats(averageMemoryUsageTracker);
+        assertAverageUsageStats(averageCpuUsageTracker);
+    }
+
+    public void testUpdateWindowSize() {
+        assertUpdateWindowSize(averageMemoryUsageTracker);
+        assertUpdateWindowSize(averageCpuUsageTracker);
+    }
+
+    private void assertAverageUsageStats(AbstractAverageUsageTracker usageTracker) {
+        usageTracker.recordUsage(1);
+        assertFalse(usageTracker.isReady());
+        usageTracker.recordUsage(2);
+        assertTrue(usageTracker.isReady());
+        assertEquals(2, usageTracker.getWindowSize());
+        assertEquals(1.5, usageTracker.getAverage(), 0.0);
+        usageTracker.recordUsage(5);
+        // ( 2 + 5 ) / 2 = 3.5
+        assertEquals(3.5, usageTracker.getAverage(), 0.0);
+    }
+
+    private void assertUpdateWindowSize(AbstractAverageUsageTracker usageTracker) {
+        usageTracker.recordUsage(1);
+        usageTracker.recordUsage(2);
+
+        assertEquals(2, usageTracker.getWindowSize());
+        assertEquals(1.5, usageTracker.getAverage(), 0.0);
+        usageTracker.recordUsage(5);
+        // ( 2 + 5 ) / 2 = 3.5
+        assertEquals(3.5, usageTracker.getAverage(), 0.0);
+
+        usageTracker.setWindowSize(new TimeValue(2000, TimeUnit.MILLISECONDS));
+        assertEquals(0, usageTracker.getWindowSize());
+        assertEquals(0.0, usageTracker.getAverage(), 0.0);
+        // verify 2000/500 = 4 is the window size and average is calculated on window size of 4
+        usageTracker.recordUsage(1);
+        usageTracker.recordUsage(2);
+        usageTracker.recordUsage(1);
+        assertFalse(usageTracker.isReady());
+        usageTracker.recordUsage(2);
+        assertTrue(usageTracker.isReady());
+        assertEquals(4, usageTracker.getWindowSize());
+        // (1 + 2 + 1 + 2 ) / 4 = 1.5
+        assertEquals(1.5, usageTracker.getAverage(), 0.0);
+        usageTracker.recordUsage(2);
+        assertTrue(usageTracker.isReady());
+        // ( 2 + 1 + 2 + 2 ) / 4 = 1.75
+        assertEquals(1.75, usageTracker.getAverage(), 0.0);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/node/resource/tracker/NodeResourceUsageTrackerTests.java b/server/src/test/java/org/opensearch/node/resource/tracker/NodeResourceUsageTrackerTests.java
new file mode 100644
index 0000000000000..1ce68b9f29062
--- /dev/null
+++ b/server/src/test/java/org/opensearch/node/resource/tracker/NodeResourceUsageTrackerTests.java
@@ -0,0 +1,96 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.node.resource.tracker;
+
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * Tests to assert resource usage trackers retrieving resource utilization averages
+ */
+public class NodeResourceUsageTrackerTests extends OpenSearchSingleNodeTestCase {
+    ThreadPool threadPool;
+
+    @Before
+    public void setup() {
+        threadPool = new TestThreadPool(getClass().getName());
+    }
+
+    @After
+    public void cleanup() {
+        ThreadPool.terminate(threadPool, 5, TimeUnit.SECONDS);
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().putNull("*"))
+                .setTransientSettings(Settings.builder().putNull("*"))
+        );
+    }
+
+    public void testStats() throws Exception {
+        Settings settings = Settings.builder()
+            .put(ResourceTrackerSettings.GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING.getKey(), new TimeValue(500, TimeUnit.MILLISECONDS))
+            .build();
+        NodeResourceUsageTracker tracker = new NodeResourceUsageTracker(
+            threadPool,
+            settings,
+            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        tracker.start();
+        /**
+         * Asserting memory utilization to be greater than 0
+         * cpu percent used is mostly 0, so skipping assertion for that
+         */
+        assertBusy(() -> assertThat(tracker.getMemoryUtilizationPercent(), greaterThan(0.0)), 5, TimeUnit.SECONDS);
+        tracker.stop();
+        tracker.close();
+    }
+
+    public void testUpdateSettings() {
+        NodeResourceUsageTracker tracker = new NodeResourceUsageTracker(
+            threadPool,
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+
+        assertEquals(tracker.getResourceTrackerSettings().getCpuWindowDuration().getSeconds(), 30);
+        assertEquals(tracker.getResourceTrackerSettings().getMemoryWindowDuration().getSeconds(), 30);
+
+        Settings settings = Settings.builder()
+            .put(ResourceTrackerSettings.GLOBAL_CPU_USAGE_AC_WINDOW_DURATION_SETTING.getKey(), "10s")
+            .build();
+        ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settings).get();
+        assertEquals(
+            "10s",
+            response.getPersistentSettings().get(ResourceTrackerSettings.GLOBAL_CPU_USAGE_AC_WINDOW_DURATION_SETTING.getKey())
+        );
+
+        Settings jvmsettings = Settings.builder()
+            .put(ResourceTrackerSettings.GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING.getKey(), "5s")
+            .build();
+        response = client().admin().cluster().prepareUpdateSettings().setPersistentSettings(jvmsettings).get();
+        assertEquals(
+            "5s",
+            response.getPersistentSettings().get(ResourceTrackerSettings.GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING.getKey())
+        );
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
index 6354cf18e8b62..a520b6278ea47 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
@@ -112,6 +112,7 @@ List adjustNodesStats(List nodesStats) {
                 nodeStats.getDiscoveryStats(),
                 nodeStats.getIngestStats(),
                 nodeStats.getAdaptiveSelectionStats(),
+                nodeStats.getResourceUsageStats(),
                 nodeStats.getScriptCacheStats(),
                 nodeStats.getIndexingPressureStats(),
                 nodeStats.getShardIndexingPressureStats(),
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index acdc5b94804c6..3c7423f73685f 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -2718,6 +2718,7 @@ public void ensureEstimatedStats() {
                     false,
                     false,
                     false,
+                    false,
                     false
                 );
                 assertThat(

From dd6355910bb92af47f78970d50ca646a66b014be Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Oct 2023 10:15:31 -0400
Subject: [PATCH 164/388] Bump org.codehaus.woodstox:stax2-api from 4.2.1 to
 4.2.2 in /plugins/repository-hdfs (#10639)

* Bump org.codehaus.woodstox:stax2-api in /plugins/repository-hdfs

Bumps [org.codehaus.woodstox:stax2-api](https://github.com/FasterXML/stax2-api) from 4.2.1 to 4.2.2.
- [Commits](https://github.com/FasterXML/stax2-api/compare/stax2-api-4.2.1...stax2-api-4.2.2)

---
updated-dependencies:
- dependency-name: org.codehaus.woodstox:stax2-api
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 

* Updating SHAs

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 CHANGELOG.md                                              | 3 ++-
 plugins/repository-hdfs/build.gradle                      | 2 +-
 plugins/repository-hdfs/licenses/stax2-api-4.2.1.jar.sha1 | 1 -
 plugins/repository-hdfs/licenses/stax2-api-4.2.2.jar.sha1 | 1 +
 4 files changed, 4 insertions(+), 3 deletions(-)
 delete mode 100644 plugins/repository-hdfs/licenses/stax2-api-4.2.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/stax2-api-4.2.2.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2211f59574718..018ad6a676bcf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -94,6 +94,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
 - Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
 - Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
+- Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -113,4 +114,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 1ce29afe0e69a..ed1f54888a26f 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -84,7 +84,7 @@ dependencies {
   api 'net.minidev:json-smart:2.5.0'
   api "io.netty:netty-all:${versions.netty}"
   implementation "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
-  implementation 'org.codehaus.woodstox:stax2-api:4.2.1'
+  implementation 'org.codehaus.woodstox:stax2-api:4.2.2'
 
   hdfsFixture project(':test:fixtures:hdfs-fixture')
   // Set the keytab files in the classpath so that we can access them from test code without the security manager
diff --git a/plugins/repository-hdfs/licenses/stax2-api-4.2.1.jar.sha1 b/plugins/repository-hdfs/licenses/stax2-api-4.2.1.jar.sha1
deleted file mode 100644
index 2c12704cdc560..0000000000000
--- a/plugins/repository-hdfs/licenses/stax2-api-4.2.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a3f7325c52240418c2ba257b103c3c550e140c83
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/stax2-api-4.2.2.jar.sha1 b/plugins/repository-hdfs/licenses/stax2-api-4.2.2.jar.sha1
new file mode 100644
index 0000000000000..b15a7ead0d016
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/stax2-api-4.2.2.jar.sha1
@@ -0,0 +1 @@
+b0d746cadea928e5264f2ea294ea9a1bf815bbde
\ No newline at end of file

From e8ba35cff433b3dc0d821d04326b2bf261beab06 Mon Sep 17 00:00:00 2001
From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>
Date: Mon, 16 Oct 2023 17:15:26 -0400
Subject: [PATCH 165/388] Remove the authenticate token method from
 TokenManager interface (#10614)

Remove the authenticate token method from TokenManager interface

Signed-off-by: Stephen Crawford 
Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>
---
 .../identity/shiro/ShiroTokenManager.java        | 16 ----------------
 .../identity/shiro/AuthTokenHandlerTests.java    |  5 -----
 .../identity/noop/NoopTokenManager.java          |  5 -----
 .../opensearch/identity/tokens/TokenManager.java |  7 -------
 4 files changed, 33 deletions(-)

diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java
index ddfb99e626718..a14215aa7655b 100644
--- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java
+++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java
@@ -10,13 +10,11 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.shiro.SecurityUtils;
 import org.apache.shiro.authc.AuthenticationToken;
 import org.apache.shiro.authc.UsernamePasswordToken;
 import org.opensearch.common.Randomness;
 import org.opensearch.identity.IdentityService;
 import org.opensearch.identity.Subject;
-import org.opensearch.identity.noop.NoopSubject;
 import org.opensearch.identity.tokens.AuthToken;
 import org.opensearch.identity.tokens.BasicAuthToken;
 import org.opensearch.identity.tokens.OnBehalfOfClaims;
@@ -88,20 +86,6 @@ public AuthToken issueServiceAccountToken(String audience) {
         return token;
     }
 
-    @Override
-    public Subject authenticateToken(AuthToken authToken) {
-        return new NoopSubject();
-    }
-
-    public boolean validateToken(AuthToken token) {
-        if (token instanceof BasicAuthToken) {
-            final BasicAuthToken basicAuthToken = (BasicAuthToken) token;
-            return basicAuthToken.getUser().equals(SecurityUtils.getSubject().toString())
-                && basicAuthToken.getPassword().equals(shiroTokenPasswordMap.get(basicAuthToken));
-        }
-        return false;
-    }
-
     public String getTokenInfo(AuthToken token) {
         if (token instanceof BasicAuthToken) {
             final BasicAuthToken basicAuthToken = (BasicAuthToken) token;
diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java
index db77ced298991..f99484083e2fb 100644
--- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java
+++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java
@@ -100,11 +100,6 @@ public void testShouldFailGetTokenInfo() {
         assertThrows(UnsupportedAuthenticationToken.class, () -> shiroAuthTokenHandler.getTokenInfo(bearerAuthToken));
     }
 
-    public void testShouldFailValidateToken() {
-        final BearerAuthToken bearerAuthToken = new BearerAuthToken("header.payload.signature");
-        assertFalse(shiroAuthTokenHandler.validateToken(bearerAuthToken));
-    }
-
     public void testShoudPassMapLookupWithToken() {
         final BasicAuthToken authToken = new BasicAuthToken("Basic dGVzdDp0ZTpzdA==");
         shiroAuthTokenHandler.getShiroTokenPasswordMap().put(authToken, "te:st");
diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java
index 1dc3a58916b5c..fa6643b7447dc 100644
--- a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java
+++ b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java
@@ -50,9 +50,4 @@ public String asAuthHeaderValue() {
             }
         };
     }
-
-    @Override
-    public Subject authenticateToken(AuthToken authToken) {
-        return null;
-    }
 }
diff --git a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java
index b4048251a06a2..972a9a1080955 100644
--- a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java
+++ b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java
@@ -30,11 +30,4 @@ public interface TokenManager {
      * @return a new auth token
      */
     public AuthToken issueServiceAccountToken(final String audience);
-
-    /**
-     * Authenticates a provided authToken
-     * @param authToken: The authToken to authenticate
-     * @return The authenticated subject
-     */
-    public Subject authenticateToken(AuthToken authToken);
 }

From a8574cd69757171732b43bb2ec91390be4324b22 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Oct 2023 18:11:45 -0400
Subject: [PATCH 166/388] Bump com.google.http-client:google-http-client from
 1.43.2 to 1.43.3 in /plugins/repository-gcs (#10635)

* Bump com.google.http-client:google-http-client

Bumps [com.google.http-client:google-http-client](https://github.com/googleapis/google-http-java-client) from 1.43.2 to 1.43.3.
- [Release notes](https://github.com/googleapis/google-http-java-client/releases)
- [Changelog](https://github.com/googleapis/google-http-java-client/blob/main/CHANGELOG.md)
- [Commits](https://github.com/googleapis/google-http-java-client/compare/v1.43.2...v1.43.3)

---
updated-dependencies:
- dependency-name: com.google.http-client:google-http-client
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 

* Updating SHAs

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 CHANGELOG.md                                                  | 1 +
 plugins/repository-gcs/build.gradle                           | 4 ++--
 .../licenses/google-http-client-1.43.2.jar.sha1               | 1 -
 .../licenses/google-http-client-1.43.3.jar.sha1               | 1 +
 .../licenses/google-http-client-appengine-1.43.2.jar.sha1     | 1 -
 .../licenses/google-http-client-appengine-1.43.3.jar.sha1     | 1 +
 6 files changed, 5 insertions(+), 4 deletions(-)
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-1.43.2.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-appengine-1.43.2.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 018ad6a676bcf..4dd19586ba263 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -95,6 +95,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
 - Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
+- Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
index da4978608a12f..1bef5146f1db9 100644
--- a/plugins/repository-gcs/build.gradle
+++ b/plugins/repository-gcs/build.gradle
@@ -75,8 +75,8 @@ dependencies {
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api 'com.google.guava:failureaccess:1.0.1'
 
-  api 'com.google.http-client:google-http-client:1.43.2'
-  api 'com.google.http-client:google-http-client-appengine:1.43.2'
+  api 'com.google.http-client:google-http-client:1.43.3'
+  api 'com.google.http-client:google-http-client-appengine:1.43.3'
   api 'com.google.http-client:google-http-client-gson:1.43.3'
   api 'com.google.http-client:google-http-client-jackson2:1.43.3'
 
diff --git a/plugins/repository-gcs/licenses/google-http-client-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.43.2.jar.sha1
deleted file mode 100644
index a576a74c62542..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-1.43.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2520469ebd8c0675f0d2aeafd2da665228320fcf
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1
new file mode 100644
index 0000000000000..800467de8bdf3
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1
@@ -0,0 +1 @@
+a758b82e55a2f5f681e289c5ed384d3dbda6f3cd
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.2.jar.sha1
deleted file mode 100644
index d8a9dba20070b..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9fb548c5264227813fd83991b94a705b0841c15f
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1
new file mode 100644
index 0000000000000..4adcca6a55902
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1
@@ -0,0 +1 @@
+09d6cbdde6ea3469a67601a811b4e83de3e68a79
\ No newline at end of file

From ec0cbb3a6efb9cccb02ebc020e45260c61a65244 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Mon, 16 Oct 2023 18:25:30 -0400
Subject: [PATCH 167/388] [BUG] Fix java.lang.SecurityException in
 repository-gcs plugin (#10642)

Signed-off-by: Andriy Redko 
---
 CHANGELOG.md                                        |  1 +
 .../repositories/gcs/GoogleCloudStorageService.java |  2 +-
 .../opensearch/repositories/gcs/SocketAccess.java   | 13 +++++++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4dd19586ba263..33c6f00fdd9ce 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
 - Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
 - [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
+- [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
 
 ### Deprecated
 
diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java
index 445e1d65f3d3e..c9ebb3acaf3e5 100644
--- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java
+++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java
@@ -228,7 +228,7 @@ StorageOptions createStorageOptions(
             }
             storageOptionsBuilder.setCredentials(serviceAccountCredentials);
         }
-        return storageOptionsBuilder.build();
+        return SocketAccess.doPrivilegedException(() -> storageOptionsBuilder.build());
     }
 
     /**
diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/SocketAccess.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/SocketAccess.java
index 197e772df30d5..35127d6ea4060 100644
--- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/SocketAccess.java
+++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/SocketAccess.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.repositories.gcs;
 
+import org.apache.logging.log4j.core.util.Throwables;
 import org.opensearch.SpecialPermission;
 import org.opensearch.common.CheckedRunnable;
 
@@ -71,4 +72,16 @@ public static void doPrivilegedVoidIOException(CheckedRunnable acti
             throw (IOException) e.getCause();
         }
     }
+
+    public static  T doPrivilegedException(PrivilegedExceptionAction operation) {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            Throwables.rethrow(e.getCause());
+            assert false : "always throws";
+            return null;
+        }
+    }
+
 }

From 4112f8d1a5594b88cf267b990a954711670a34cf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Oct 2023 19:25:28 -0400
Subject: [PATCH 168/388] Bump com.squareup.okio:okio from 3.5.0 to 3.6.0 in
 /test/fixtures/hdfs-fixture (#10637)

* Bump com.squareup.okio:okio in /test/fixtures/hdfs-fixture

Bumps [com.squareup.okio:okio](https://github.com/square/okio) from 3.5.0 to 3.6.0.
- [Changelog](https://github.com/square/okio/blob/master/CHANGELOG.md)
- [Commits](https://github.com/square/okio/compare/parent-3.5.0...parent-3.6.0)

---
updated-dependencies:
- dependency-name: com.squareup.okio:okio
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 CHANGELOG.md                            | 1 +
 test/fixtures/hdfs-fixture/build.gradle | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33c6f00fdd9ce..46b3fc7852ebc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -96,6 +96,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
+- Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 7de98c0986b87..13345fcb20de1 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -71,6 +71,6 @@ dependencies {
   runtimeOnly("com.squareup.okhttp3:okhttp:4.11.0") {
     exclude group: "com.squareup.okio"
   }
-  runtimeOnly "com.squareup.okio:okio:3.5.0"
+  runtimeOnly "com.squareup.okio:okio:3.6.0"
   runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.5"
 }

From 8bc6791f6a3bd58b4ee25bf5ae61504570c8d424 Mon Sep 17 00:00:00 2001
From: Bharathwaj G 
Date: Tue, 17 Oct 2023 10:57:14 +0530
Subject: [PATCH 169/388] Enhancing FS stats to include read/write time, queue
 size and IO time (#10541)

* Enhancing FS stats to include read / write time, io time and queue size

Signed-off-by: Bharathwaj G 
---
 CHANGELOG.md                                  |   1 +
 .../org/opensearch/monitor/fs/FsInfo.java     | 178 +++++++++++++++++-
 .../org/opensearch/monitor/fs/FsProbe.java    |  29 +++
 .../cluster/node/stats/NodeStatsTests.java    |  12 ++
 .../monitor/fs/DeviceStatsTests.java          |  19 +-
 .../opensearch/monitor/fs/FsProbeTests.java   |  51 +++++
 6 files changed, 285 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 46b3fc7852ebc..a02b5f4e7242e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
+- [Admission control] Add enhancements to FS stats to include read/write time, queue size and IO time ([#10541](https://github.com/opensearch-project/OpenSearch/pull/10541))
 - [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
 
 ### Dependencies
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
index 114702ff0d351..4e2e9f280d765 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
@@ -235,6 +235,14 @@ public static class DeviceStats implements Writeable, ToXContentFragment {
         final long previousWritesCompleted;
         final long currentSectorsWritten;
         final long previousSectorsWritten;
+        final long currentReadTime;
+        final long previousReadTime;
+        final long currentWriteTime;
+        final long previousWriteTime;
+        final long currentQueueSize;
+        final long previousQueueSize;
+        final long currentIOTime;
+        final long previousIOTime;
 
         public DeviceStats(
             final int majorDeviceNumber,
@@ -244,6 +252,10 @@ public DeviceStats(
             final long currentSectorsRead,
             final long currentWritesCompleted,
             final long currentSectorsWritten,
+            final long currentReadTime,
+            final long currentWriteTime,
+            final long currrentQueueSize,
+            final long currentIOTime,
             final DeviceStats previousDeviceStats
         ) {
             this(
@@ -257,7 +269,15 @@ public DeviceStats(
                 currentSectorsRead,
                 previousDeviceStats != null ? previousDeviceStats.currentSectorsRead : -1,
                 currentWritesCompleted,
-                previousDeviceStats != null ? previousDeviceStats.currentWritesCompleted : -1
+                previousDeviceStats != null ? previousDeviceStats.currentWritesCompleted : -1,
+                currentReadTime,
+                previousDeviceStats != null ? previousDeviceStats.currentReadTime : -1,
+                currentWriteTime,
+                previousDeviceStats != null ? previousDeviceStats.currentWriteTime : -1,
+                currrentQueueSize,
+                previousDeviceStats != null ? previousDeviceStats.currentQueueSize : -1,
+                currentIOTime,
+                previousDeviceStats != null ? previousDeviceStats.currentIOTime : -1
             );
         }
 
@@ -272,7 +292,15 @@ private DeviceStats(
             final long currentSectorsRead,
             final long previousSectorsRead,
             final long currentWritesCompleted,
-            final long previousWritesCompleted
+            final long previousWritesCompleted,
+            final long currentReadTime,
+            final long previousReadTime,
+            final long currentWriteTime,
+            final long previousWriteTime,
+            final long currentQueueSize,
+            final long previousQueueSize,
+            final long currentIOTime,
+            final long previousIOTime
         ) {
             this.majorDeviceNumber = majorDeviceNumber;
             this.minorDeviceNumber = minorDeviceNumber;
@@ -285,6 +313,14 @@ private DeviceStats(
             this.previousSectorsRead = previousSectorsRead;
             this.currentSectorsWritten = currentSectorsWritten;
             this.previousSectorsWritten = previousSectorsWritten;
+            this.currentReadTime = currentReadTime;
+            this.previousReadTime = previousReadTime;
+            this.currentWriteTime = currentWriteTime;
+            this.previousWriteTime = previousWriteTime;
+            this.currentQueueSize = currentQueueSize;
+            this.previousQueueSize = previousQueueSize;
+            this.currentIOTime = currentIOTime;
+            this.previousIOTime = previousIOTime;
         }
 
         public DeviceStats(StreamInput in) throws IOException {
@@ -299,6 +335,25 @@ public DeviceStats(StreamInput in) throws IOException {
             previousSectorsRead = in.readLong();
             currentSectorsWritten = in.readLong();
             previousSectorsWritten = in.readLong();
+            if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+                currentReadTime = in.readLong();
+                previousReadTime = in.readLong();
+                currentWriteTime = in.readLong();
+                previousWriteTime = in.readLong();
+                currentQueueSize = in.readLong();
+                previousQueueSize = in.readLong();
+                currentIOTime = in.readLong();
+                previousIOTime = in.readLong();
+            } else {
+                currentReadTime = 0;
+                previousReadTime = 0;
+                currentWriteTime = 0;
+                previousWriteTime = 0;
+                currentQueueSize = 0;
+                previousQueueSize = 0;
+                currentIOTime = 0;
+                previousIOTime = 0;
+            }
         }
 
         @Override
@@ -314,6 +369,16 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(previousSectorsRead);
             out.writeLong(currentSectorsWritten);
             out.writeLong(previousSectorsWritten);
+            if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+                out.writeLong(currentReadTime);
+                out.writeLong(previousReadTime);
+                out.writeLong(currentWriteTime);
+                out.writeLong(previousWriteTime);
+                out.writeLong(currentQueueSize);
+                out.writeLong(previousQueueSize);
+                out.writeLong(currentIOTime);
+                out.writeLong(previousIOTime);
+            }
         }
 
         public long operations() {
@@ -346,6 +411,39 @@ public long writeKilobytes() {
             return (currentSectorsWritten - previousSectorsWritten) / 2;
         }
 
+        /**
+         * Total time taken for all read operations
+         */
+        public long readTime() {
+            if (previousReadTime == -1) return -1;
+            return currentReadTime - previousReadTime;
+        }
+
+        /**
+         * Total time taken for all write operations
+         */
+        public long writeTime() {
+            if (previousWriteTime == -1) return -1;
+            return currentWriteTime - previousWriteTime;
+        }
+
+        /**
+         * Queue size based on weighted time spent doing I/Os
+         */
+        public long queueSize() {
+            if (previousQueueSize == -1) return -1;
+            return currentQueueSize - previousQueueSize;
+        }
+
+        /**
+         * Total time spent doing I/Os
+         */
+        public long ioTimeInMillis() {
+            if (previousIOTime == -1) return -1;
+
+            return (currentIOTime - previousIOTime);
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field("device_name", deviceName);
@@ -354,9 +452,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(IoStats.WRITE_OPERATIONS, writeOperations());
             builder.field(IoStats.READ_KILOBYTES, readKilobytes());
             builder.field(IoStats.WRITE_KILOBYTES, writeKilobytes());
+            builder.field(IoStats.READ_TIME, readTime());
+            builder.field(IoStats.WRITE_TIME, writeTime());
+            builder.field(IoStats.QUEUE_SIZE, queueSize());
+            builder.field(IoStats.IO_TIME_MS, ioTimeInMillis());
             return builder;
         }
-
     }
 
     /**
@@ -371,6 +472,10 @@ public static class IoStats implements Writeable, ToXContentFragment {
         private static final String WRITE_OPERATIONS = "write_operations";
         private static final String READ_KILOBYTES = "read_kilobytes";
         private static final String WRITE_KILOBYTES = "write_kilobytes";
+        private static final String READ_TIME = "read_time";
+        private static final String WRITE_TIME = "write_time";
+        private static final String QUEUE_SIZE = "queue_size";
+        private static final String IO_TIME_MS = "io_time_in_millis";
 
         final DeviceStats[] devicesStats;
         final long totalOperations;
@@ -378,6 +483,10 @@ public static class IoStats implements Writeable, ToXContentFragment {
         final long totalWriteOperations;
         final long totalReadKilobytes;
         final long totalWriteKilobytes;
+        final long totalReadTime;
+        final long totalWriteTime;
+        final long totalQueueSize;
+        final long totalIOTimeInMillis;
 
         public IoStats(final DeviceStats[] devicesStats) {
             this.devicesStats = devicesStats;
@@ -386,18 +495,30 @@ public IoStats(final DeviceStats[] devicesStats) {
             long totalWriteOperations = 0;
             long totalReadKilobytes = 0;
             long totalWriteKilobytes = 0;
+            long totalReadTime = 0;
+            long totalWriteTime = 0;
+            long totalQueueSize = 0;
+            long totalIOTimeInMillis = 0;
             for (DeviceStats deviceStats : devicesStats) {
                 totalOperations += deviceStats.operations() != -1 ? deviceStats.operations() : 0;
                 totalReadOperations += deviceStats.readOperations() != -1 ? deviceStats.readOperations() : 0;
                 totalWriteOperations += deviceStats.writeOperations() != -1 ? deviceStats.writeOperations() : 0;
                 totalReadKilobytes += deviceStats.readKilobytes() != -1 ? deviceStats.readKilobytes() : 0;
                 totalWriteKilobytes += deviceStats.writeKilobytes() != -1 ? deviceStats.writeKilobytes() : 0;
+                totalReadTime += deviceStats.readTime() != -1 ? deviceStats.readTime() : 0;
+                totalWriteTime += deviceStats.writeTime() != -1 ? deviceStats.writeTime() : 0;
+                totalQueueSize += deviceStats.queueSize() != -1 ? deviceStats.queueSize() : 0;
+                totalIOTimeInMillis += deviceStats.ioTimeInMillis() != -1 ? deviceStats.ioTimeInMillis() : 0;
             }
             this.totalOperations = totalOperations;
             this.totalReadOperations = totalReadOperations;
             this.totalWriteOperations = totalWriteOperations;
             this.totalReadKilobytes = totalReadKilobytes;
             this.totalWriteKilobytes = totalWriteKilobytes;
+            this.totalReadTime = totalReadTime;
+            this.totalWriteTime = totalWriteTime;
+            this.totalQueueSize = totalQueueSize;
+            this.totalIOTimeInMillis = totalIOTimeInMillis;
         }
 
         public IoStats(StreamInput in) throws IOException {
@@ -412,6 +533,17 @@ public IoStats(StreamInput in) throws IOException {
             this.totalWriteOperations = in.readLong();
             this.totalReadKilobytes = in.readLong();
             this.totalWriteKilobytes = in.readLong();
+            if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+                this.totalReadTime = in.readLong();
+                this.totalWriteTime = in.readLong();
+                this.totalQueueSize = in.readLong();
+                this.totalIOTimeInMillis = in.readLong();
+            } else {
+                this.totalReadTime = 0;
+                this.totalWriteTime = 0;
+                this.totalQueueSize = 0;
+                this.totalIOTimeInMillis = 0;
+            }
         }
 
         @Override
@@ -425,6 +557,12 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(totalWriteOperations);
             out.writeLong(totalReadKilobytes);
             out.writeLong(totalWriteKilobytes);
+            if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+                out.writeLong(totalReadTime);
+                out.writeLong(totalWriteTime);
+                out.writeLong(totalQueueSize);
+                out.writeLong(totalIOTimeInMillis);
+            }
         }
 
         public DeviceStats[] getDevicesStats() {
@@ -451,6 +589,34 @@ public long getTotalWriteKilobytes() {
             return totalWriteKilobytes;
         }
 
+        /**
+         * Sum of read time across all devices
+         */
+        public long getTotalReadTime() {
+            return totalReadTime;
+        }
+
+        /**
+         * Sum of write time across all devices
+         */
+        public long getTotalWriteTime() {
+            return totalWriteTime;
+        }
+
+        /**
+         * Sum of queue size across all devices
+         */
+        public long getTotalQueueSize() {
+            return totalQueueSize;
+        }
+
+        /**
+         * Sum of IO time across all devices
+         */
+        public long getTotalIOTimeMillis() {
+            return totalIOTimeInMillis;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             if (devicesStats.length > 0) {
@@ -468,11 +634,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(WRITE_OPERATIONS, totalWriteOperations);
                 builder.field(READ_KILOBYTES, totalReadKilobytes);
                 builder.field(WRITE_KILOBYTES, totalWriteKilobytes);
+
+                builder.field(READ_TIME, totalReadTime);
+                builder.field(WRITE_TIME, totalWriteTime);
+                builder.field(QUEUE_SIZE, totalQueueSize);
+                builder.field(IO_TIME_MS, totalIOTimeInMillis);
                 builder.endObject();
             }
             return builder;
         }
-
     }
 
     private final long timestamp;
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
index e20d84cd9763e..f4731a4a34373 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
@@ -109,6 +109,25 @@ final FsInfo.IoStats ioStats(final Set> devicesNumbers,
 
             List devicesStats = new ArrayList<>();
 
+            /**
+             * The /proc/diskstats file displays the I/O statistics of block devices.
+             * Each line contains the following 14 fields: ( + additional fields )
+            *
+            * 1  major number
+            * 2  minor number
+            * 3  device name
+            * 4  reads completed successfully
+            * 5  reads merged
+            * 6  sectors read
+            * 7  time spent reading (ms)
+            * 8  writes completed
+            * 9  writes merged
+            * 10  sectors written
+            * 11  time spent writing (ms)
+            * 12  I/Os currently in progress
+            * 13  time spent doing I/Os (ms) ---- IO use percent
+            * 14  weighted time spent doing I/Os (ms) ---- Queue size
+             */
             List lines = readProcDiskStats();
             if (!lines.isEmpty()) {
                 for (String line : lines) {
@@ -123,6 +142,12 @@ final FsInfo.IoStats ioStats(final Set> devicesNumbers,
                     final long sectorsRead = Long.parseLong(fields[5]);
                     final long writesCompleted = Long.parseLong(fields[7]);
                     final long sectorsWritten = Long.parseLong(fields[9]);
+                    // readTime and writeTime calculates the total read/write time taken for each request to complete
+                    // ioTime calculates actual time queue and disks are busy
+                    final long readTime = Long.parseLong(fields[6]);
+                    final long writeTime = Long.parseLong(fields[10]);
+                    final long ioTime = fields.length > 12 ? Long.parseLong(fields[12]) : 0;
+                    final long queueSize = fields.length > 13 ? Long.parseLong(fields[13]) : 0;
                     final FsInfo.DeviceStats deviceStats = new FsInfo.DeviceStats(
                         majorDeviceNumber,
                         minorDeviceNumber,
@@ -131,6 +156,10 @@ final FsInfo.IoStats ioStats(final Set> devicesNumbers,
                         sectorsRead,
                         writesCompleted,
                         sectorsWritten,
+                        readTime,
+                        writeTime,
+                        queueSize,
+                        ioTime,
                         deviceMap.get(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))
                     );
                     devicesStats.add(deviceStats);
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 7a1b6f113d0e8..4b865383ee007 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -284,6 +284,10 @@ public void testSerialization() throws IOException {
                     assertEquals(ioStats.getTotalReadOperations(), deserializedIoStats.getTotalReadOperations());
                     assertEquals(ioStats.getTotalWriteKilobytes(), deserializedIoStats.getTotalWriteKilobytes());
                     assertEquals(ioStats.getTotalWriteOperations(), deserializedIoStats.getTotalWriteOperations());
+                    assertEquals(ioStats.getTotalReadTime(), deserializedIoStats.getTotalReadTime());
+                    assertEquals(ioStats.getTotalWriteTime(), deserializedIoStats.getTotalWriteTime());
+                    assertEquals(ioStats.getTotalQueueSize(), deserializedIoStats.getTotalQueueSize());
+                    assertEquals(ioStats.getTotalIOTimeMillis(), deserializedIoStats.getTotalIOTimeMillis());
                     assertEquals(ioStats.getDevicesStats().length, deserializedIoStats.getDevicesStats().length);
                     for (int i = 0; i < ioStats.getDevicesStats().length; i++) {
                         FsInfo.DeviceStats deviceStats = ioStats.getDevicesStats()[i];
@@ -645,6 +649,10 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
                         randomNonNegativeLong(),
                         randomNonNegativeLong(),
                         randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
                         null
                     );
                 deviceStatsArray[i] = new FsInfo.DeviceStats(
@@ -655,6 +663,10 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
                     randomNonNegativeLong(),
                     randomNonNegativeLong(),
                     randomNonNegativeLong(),
+                    randomNonNegativeLong(),
+                    randomNonNegativeLong(),
+                    randomNonNegativeLong(),
+                    randomNonNegativeLong(),
                     previousDeviceStats
                 );
             }
diff --git a/server/src/test/java/org/opensearch/monitor/fs/DeviceStatsTests.java b/server/src/test/java/org/opensearch/monitor/fs/DeviceStatsTests.java
index 0fd039b84e887..0059f8e215f2e 100644
--- a/server/src/test/java/org/opensearch/monitor/fs/DeviceStatsTests.java
+++ b/server/src/test/java/org/opensearch/monitor/fs/DeviceStatsTests.java
@@ -46,7 +46,12 @@ public void testDeviceStats() {
         final int sectorsRead = randomIntBetween(8 * readsCompleted, 16 * readsCompleted);
         final int writesCompleted = randomIntBetween(1, 1 << 16);
         final int sectorsWritten = randomIntBetween(8 * writesCompleted, 16 * writesCompleted);
-
+        final int readTime = randomIntBetween(1, 1 << 16);
+        ;
+        final int writeTime = randomIntBetween(1, 1 << 16);
+        ;
+        final int queueSize = randomIntBetween(1, 1 << 16);
+        final int ioTime = randomIntBetween(1, 1 << 16);
         FsInfo.DeviceStats previous = new FsInfo.DeviceStats(
             majorDeviceNumber,
             minorDeviceNumber,
@@ -55,6 +60,10 @@ public void testDeviceStats() {
             sectorsRead,
             writesCompleted,
             sectorsWritten,
+            readTime,
+            writeTime,
+            queueSize,
+            ioTime,
             null
         );
         FsInfo.DeviceStats current = new FsInfo.DeviceStats(
@@ -65,6 +74,10 @@ public void testDeviceStats() {
             sectorsRead + 16384,
             writesCompleted + 2048,
             sectorsWritten + 32768,
+            readTime + 500,
+            writeTime + 100,
+            queueSize + 20,
+            ioTime + 8192,
             previous
         );
         assertThat(current.operations(), equalTo(1024L + 2048L));
@@ -72,6 +85,10 @@ public void testDeviceStats() {
         assertThat(current.writeOperations(), equalTo(2048L));
         assertThat(current.readKilobytes(), equalTo(16384L / 2));
         assertThat(current.writeKilobytes(), equalTo(32768L / 2));
+        assertEquals(500, current.readTime());
+        assertEquals(100, current.writeTime());
+        assertEquals(20, current.queueSize());
+        assertEquals(8192, current.ioTimeInMillis());
     }
 
 }
diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java
index 686a624d988d7..59a888c665be7 100644
--- a/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java
+++ b/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java
@@ -91,6 +91,14 @@ public void testFsInfo() throws IOException {
                     assertThat(deviceStats.previousWritesCompleted, equalTo(-1L));
                     assertThat(deviceStats.currentSectorsWritten, greaterThanOrEqualTo(0L));
                     assertThat(deviceStats.previousSectorsWritten, equalTo(-1L));
+                    assertThat(deviceStats.currentReadTime, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousReadTime, greaterThanOrEqualTo(-1L));
+                    assertThat(deviceStats.currentWriteTime, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousWriteTime, greaterThanOrEqualTo(-1L));
+                    assertThat(deviceStats.currentQueueSize, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousQueueSize, greaterThanOrEqualTo(-1L));
+                    assertThat(deviceStats.currentIOTime, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousIOTime, greaterThanOrEqualTo(-1L));
                 }
             } else {
                 assertNull(stats.getIoStats());
@@ -243,6 +251,16 @@ List readProcDiskStats() throws IOException {
         assertThat(first.devicesStats[0].previousWritesCompleted, equalTo(-1L));
         assertThat(first.devicesStats[0].currentSectorsWritten, equalTo(118857776L));
         assertThat(first.devicesStats[0].previousSectorsWritten, equalTo(-1L));
+
+        assertEquals(33457, first.devicesStats[0].currentReadTime);
+        assertEquals(-1, first.devicesStats[0].previousReadTime);
+        assertEquals(18730966, first.devicesStats[0].currentWriteTime);
+        assertEquals(-1, first.devicesStats[0].previousWriteTime);
+        assertEquals(18767169, first.devicesStats[0].currentQueueSize);
+        assertEquals(-1, first.devicesStats[0].previousQueueSize);
+        assertEquals(1918440, first.devicesStats[0].currentIOTime);
+        assertEquals(-1, first.devicesStats[0].previousIOTime);
+
         assertThat(first.devicesStats[1].majorDeviceNumber, equalTo(253));
         assertThat(first.devicesStats[1].minorDeviceNumber, equalTo(2));
         assertThat(first.devicesStats[1].deviceName, equalTo("dm-2"));
@@ -255,6 +273,15 @@ List readProcDiskStats() throws IOException {
         assertThat(first.devicesStats[1].currentSectorsWritten, equalTo(64126096L));
         assertThat(first.devicesStats[1].previousSectorsWritten, equalTo(-1L));
 
+        assertEquals(49312, first.devicesStats[1].currentReadTime);
+        assertEquals(-1, first.devicesStats[1].previousReadTime);
+        assertEquals(33730596, first.devicesStats[1].currentWriteTime);
+        assertEquals(-1, first.devicesStats[1].previousWriteTime);
+        assertEquals(33781827, first.devicesStats[1].currentQueueSize);
+        assertEquals(-1, first.devicesStats[1].previousQueueSize);
+        assertEquals(1058193, first.devicesStats[1].currentIOTime);
+        assertEquals(-1, first.devicesStats[1].previousIOTime);
+
         diskStats.set(
             Arrays.asList(
                 " 259       0 nvme0n1 336870 0 7928397 82876 10264393 0 182986405 52451610 0 2971042 52536492",
@@ -281,6 +308,16 @@ List readProcDiskStats() throws IOException {
         assertThat(second.devicesStats[0].previousWritesCompleted, equalTo(8398869L));
         assertThat(second.devicesStats[0].currentSectorsWritten, equalTo(118857776L));
         assertThat(second.devicesStats[0].previousSectorsWritten, equalTo(118857776L));
+
+        assertEquals(33464, second.devicesStats[0].currentReadTime);
+        assertEquals(33457, second.devicesStats[0].previousReadTime);
+        assertEquals(18730966, second.devicesStats[0].currentWriteTime);
+        assertEquals(18730966, second.devicesStats[0].previousWriteTime);
+        assertEquals(18767176, second.devicesStats[0].currentQueueSize);
+        assertEquals(18767169, second.devicesStats[0].previousQueueSize);
+        assertEquals(1918444, second.devicesStats[0].currentIOTime);
+        assertEquals(1918440, second.devicesStats[0].previousIOTime);
+
         assertThat(second.devicesStats[1].majorDeviceNumber, equalTo(253));
         assertThat(second.devicesStats[1].minorDeviceNumber, equalTo(2));
         assertThat(second.devicesStats[1].deviceName, equalTo("dm-2"));
@@ -293,11 +330,25 @@ List readProcDiskStats() throws IOException {
         assertThat(second.devicesStats[1].currentSectorsWritten, equalTo(64128568L));
         assertThat(second.devicesStats[1].previousSectorsWritten, equalTo(64126096L));
 
+        assertEquals(49369, second.devicesStats[1].currentReadTime);
+        assertEquals(49312, second.devicesStats[1].previousReadTime);
+        assertEquals(33730766, second.devicesStats[1].currentWriteTime);
+        assertEquals(33730596, second.devicesStats[1].previousWriteTime);
+        assertEquals(33781827, first.devicesStats[1].currentQueueSize);
+        assertEquals(-1L, first.devicesStats[1].previousQueueSize);
+        assertEquals(1058193, first.devicesStats[1].currentIOTime);
+        assertEquals(-1L, first.devicesStats[1].previousIOTime);
+
         assertThat(second.totalOperations, equalTo(575L));
         assertThat(second.totalReadOperations, equalTo(261L));
         assertThat(second.totalWriteOperations, equalTo(314L));
         assertThat(second.totalReadKilobytes, equalTo(2392L));
         assertThat(second.totalWriteKilobytes, equalTo(1236L));
+
+        assertEquals(64, second.totalReadTime);
+        assertEquals(170, second.totalWriteTime);
+        assertEquals(236, second.totalQueueSize);
+        assertEquals(158, second.totalIOTimeInMillis);
     }
 
     public void testAdjustForHugeFilesystems() throws Exception {

From d46a012c9fb01f7b3fb58a1c8fff1e352fe9efb8 Mon Sep 17 00:00:00 2001
From: David Z <38449481+dzane17@users.noreply.github.com>
Date: Tue, 17 Oct 2023 08:29:30 -0700
Subject: [PATCH 170/388] Fix flaky testEqualsAndHashcode in SearchRequestTests
 (#10649)

Signed-off-by: David Zane 
---
 .../java/org/opensearch/action/search/SearchRequestTests.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
index cdd0ea863ce37..f025e3a63b9bf 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
@@ -244,7 +244,9 @@ private SearchRequest mutate(SearchRequest searchRequest) {
         );
         mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder)));
         mutators.add(() -> mutation.setCcsMinimizeRoundtrips(searchRequest.isCcsMinimizeRoundtrips() == false));
-        mutators.add(() -> mutation.setPhaseTook(searchRequest.isPhaseTook() == false));
+        mutators.add(
+            () -> mutation.setPhaseTook(searchRequest.isPhaseTook() == null ? randomBoolean() : searchRequest.isPhaseTook() == false)
+        );
         mutators.add(
             () -> mutation.setCancelAfterTimeInterval(
                 searchRequest.getCancelAfterTimeInterval() != null

From 7b62e2f69ba766991bde166ed02dc373e6e39fa6 Mon Sep 17 00:00:00 2001
From: gaobinlong 
Date: Wed, 18 Oct 2023 02:18:57 +0800
Subject: [PATCH 171/388] Fix dissect ingest processor parsing empty brackets
 failed (#9255)

* Fix dissect ingest processor parsing empty brackets failed

Signed-off-by: Gao Binlong 

* Modify change log

Signed-off-by: Gao Binlong 

* Modify change log

Signed-off-by: Gao Binlong 

* Add assertion

Signed-off-by: Gao Binlong 

---------

Signed-off-by: Gao Binlong 
Signed-off-by: Daniel (dB.) Doubrovkine 
Co-authored-by: Daniel (dB.) Doubrovkine 
---
 CHANGELOG.md                                  |  1 +
 .../org/opensearch/dissect/DissectParser.java | 11 +++++-
 .../ingest/common/DissectProcessorTests.java  | 24 +++++++++++++
 .../test/ingest/200_dissect_processor.yml     | 35 +++++++++++++++++++
 4 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a02b5f4e7242e..284244de08829 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -111,6 +111,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix failure in dissect ingest processor parsing empty brackets ([#9225](https://github.com/opensearch-project/OpenSearch/pull/9255))
 - Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
 - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
 - Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
diff --git a/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java
index b6dc0ceb1028f..828d4b7de450e 100644
--- a/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java
+++ b/libs/dissect/src/main/java/org/opensearch/dissect/DissectParser.java
@@ -231,7 +231,10 @@ public Map parse(String inputString) {
             int lookAheadMatches;
             // start walking the input string byte by byte, look ahead for matches where needed
             // if a match is found jump forward to the end of the match
-            for (; i < input.length; i++) {
+            while (i < input.length) {
+                // start is only used to record the value of i
+                int start = i;
+
                 lookAheadMatches = 0;
                 // potential match between delimiter and input string
                 if (delimiter.length > 0 && input[i] == delimiter[0]) {
@@ -283,8 +286,14 @@ public Map parse(String inputString) {
                         delimiter = dissectPair.getDelimiter().getBytes(StandardCharsets.UTF_8);
                         // i is always one byte after the last found delimiter, aka the start of the next value
                         valueStart = i;
+                    } else {
+                        i++;
                     }
+                } else {
+                    i++;
                 }
+                // i should change anyway
+                assert (i != start);
             }
             // the last key, grab the rest of the input (unless consecutive delimiters already grabbed the last key)
             // and there is no trailing delimiter
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DissectProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DissectProcessorTests.java
index ca0c0df40f009..e42a1147825d1 100644
--- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DissectProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DissectProcessorTests.java
@@ -155,4 +155,28 @@ public void testNullValueWithOutIgnoreMissing() {
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
     }
+
+    public void testMatchEmptyBrackets() {
+        IngestDocument ingestDocument = new IngestDocument(
+            "_index",
+            "_id",
+            null,
+            null,
+            null,
+            Collections.singletonMap("message", "[foo],[bar],[]")
+        );
+        DissectProcessor dissectProcessor = new DissectProcessor("", null, "message", "[%{a}],[%{b}],[%{c}]", "", true);
+        dissectProcessor.execute(ingestDocument);
+        assertEquals("foo", ingestDocument.getFieldValue("a", String.class));
+        assertEquals("bar", ingestDocument.getFieldValue("b", String.class));
+        assertEquals("", ingestDocument.getFieldValue("c", String.class));
+
+        ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("message", "{}{}{}{baz}"));
+        dissectProcessor = new DissectProcessor("", null, "message", "{%{a}}{%{b}}{%{c}}{%{d}}", "", true);
+        dissectProcessor.execute(ingestDocument);
+        assertEquals("", ingestDocument.getFieldValue("a", String.class));
+        assertEquals("", ingestDocument.getFieldValue("b", String.class));
+        assertEquals("", ingestDocument.getFieldValue("c", String.class));
+        assertEquals("baz", ingestDocument.getFieldValue("d", String.class));
+    }
 }
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_dissect_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_dissect_processor.yml
index 916a7fe656cc2..d90e5fbf2362b 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_dissect_processor.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_dissect_processor.yml
@@ -84,3 +84,38 @@ teardown:
             }
           ]
         }
+
+---
+"Test dissect processor can match empty brackets":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "dissect" : {
+                  "field" : "message",
+                  "pattern" : "[%{a}][%{b}][%{c}]"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {message: "[foo][bar][]"}
+
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source.message: "[foo][bar][]" }
+  - match: { _source.a: "foo" }
+  - match: { _source.b: "bar" }
+  - match: { _source.c: "" }

From a36ab39a3e7da385ccd3651313b1e953a87fae7a Mon Sep 17 00:00:00 2001
From: Gagan Juneja 
Date: Wed, 18 Oct 2023 03:42:59 +0530
Subject: [PATCH 172/388] Add telemetry tracer/metric enable flag and integ
 test (#10395)

* Add telemetry tracer/metric enable flag and integ test

Signed-off-by: Gagan Juneja 

* Add Changelog

Signed-off-by: Gagan Juneja 

* Fix compilation issue

Signed-off-by: Gagan Juneja 

* Empty-Commit

Signed-off-by: Gagan Juneja 

* Add component flag to traceable wrappers

Signed-off-by: Gagan Juneja 

* Address review comment

Signed-off-by: Gagan Juneja 

* Address review comment

Signed-off-by: Gagan Juneja 

* Address review comment

Signed-off-by: Gagan Juneja 

* Address review comment

Signed-off-by: Gagan Juneja 

* Address review comment

Signed-off-by: Gagan Juneja 

* Address review comment

Signed-off-by: Gagan Juneja 

---------

Signed-off-by: Gagan Juneja 
Signed-off-by: Gagan Juneja 
Co-authored-by: Gagan Juneja 
---
 CHANGELOG.md                                  |  1 +
 .../telemetry/metrics/MetricsTelemetry.java   |  4 +-
 .../telemetry/tracing/DefaultTracer.java      |  5 +
 .../opensearch/telemetry/tracing/Tracer.java  |  6 ++
 .../telemetry/tracing/noop/NoopTracer.java    |  5 +
 .../telemetry/tracing/DefaultTracerTests.java |  1 +
 .../IntegrationTestOTelTelemetryPlugin.java   |  5 +-
 .../InMemorySingletonMetricsExporter.java     | 65 ++++++++++++
 .../TelemetryMetricsDisabledSanityIT.java     | 62 ++++++++++++
 .../TelemetryMetricsEnabledSanityIT.java      | 99 +++++++++++++++++++
 .../TelemetryTracerDisabledSanityIT.java      |  1 +
 .../TelemetryTracerEnabledSanityIT.java       |  5 +-
 .../telemetry/OTelTelemetryPlugin.java        | 27 +++--
 .../metrics/OTelMetricsTelemetry.java         |  9 +-
 .../telemetry/tracing/OTelTelemetry.java      | 23 +++--
 .../tracing/OTelTracingTelemetry.java         | 15 +--
 .../metrics/OTelMetricsTelemetryTests.java    | 20 +++-
 .../tracing/OTelTracingTelemetryTests.java    | 21 +++-
 .../common/settings/ClusterSettings.java      |  8 +-
 .../main/java/org/opensearch/node/Node.java   | 21 +++-
 .../telemetry/TelemetrySettings.java          | 27 +++++
 .../telemetry/tracing/WrappedTracer.java      |  5 +
 .../channels/TraceableHttpChannel.java        |  3 +-
 .../channels/TraceableRestChannel.java        |  3 +-
 .../TraceableTcpTransportChannel.java         |  3 +-
 .../TraceableTransportResponseHandler.java    |  3 +-
 .../listener/TraceableActionListener.java     |  3 +-
 .../metrics/MetricsRegistryFactoryTests.java  | 12 +++
 .../telemetry/tracing/TracerFactoryTests.java | 12 +++
 .../telemetry/tracing/WrappedTracerTests.java |  5 +-
 .../test/OpenSearchIntegTestCase.java         |  1 +
 .../test/OpenSearchSingleNodeTestCase.java    |  2 +
 .../test/telemetry/MockTelemetry.java         |  5 +-
 33 files changed, 429 insertions(+), 58 deletions(-)
 rename plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/{tracing => }/IntegrationTestOTelTelemetryPlugin.java (85%)
 create mode 100644 plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/InMemorySingletonMetricsExporter.java
 create mode 100644 plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsDisabledSanityIT.java
 create mode 100644 plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsEnabledSanityIT.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 284244de08829..e379002d254ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -105,6 +105,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
 - [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
 - [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
+- Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 
 ### Deprecated
 
diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java
index 2f70c28efb1cd..fb3dec8152b4f 100644
--- a/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java
+++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java
@@ -10,14 +10,12 @@
 
 import org.opensearch.common.annotation.ExperimentalApi;
 
-import java.io.Closeable;
-
 /**
  * Interface for metrics telemetry providers
   *
  * @opensearch.experimental
  */
 @ExperimentalApi
-public interface MetricsTelemetry extends MetricsRegistry, Closeable {
+public interface MetricsTelemetry extends MetricsRegistry {
 
 }
diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java
index 79b7e4aca6c2f..a3bb64ea392a9 100644
--- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java
+++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java
@@ -85,6 +85,11 @@ public SpanScope withSpanInScope(Span span) {
         return DefaultSpanScope.create(span, tracerContextStorage).attach();
     }
 
+    @Override
+    public boolean isRecording() {
+        return true;
+    }
+
     private Span createSpan(SpanCreationContext spanCreationContext, Span parentSpan) {
         return tracingTelemetry.createSpan(spanCreationContext, parentSpan);
     }
diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java
index e6d4878a5e833..8257d251e9560 100644
--- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java
+++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Tracer.java
@@ -53,4 +53,10 @@ public interface Tracer extends HttpTracer, Closeable {
      */
     SpanScope withSpanInScope(Span span);
 
+    /**
+     * Tells if the traces are being recorded or not
+     * @return boolean
+     */
+    boolean isRecording();
+
 }
diff --git a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java
index c073e8d3e766f..50452ff5fe3b4 100644
--- a/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java
+++ b/libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/noop/NoopTracer.java
@@ -54,6 +54,11 @@ public SpanScope withSpanInScope(Span span) {
         return SpanScope.NO_OP;
     }
 
+    @Override
+    public boolean isRecording() {
+        return false;
+    }
+
     @Override
     public void close() {
 
diff --git a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java
index 0a717e993cb81..2a791f1ae4164 100644
--- a/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java
+++ b/libs/telemetry/src/test/java/org/opensearch/telemetry/tracing/DefaultTracerTests.java
@@ -62,6 +62,7 @@ public void testCreateSpan() {
 
         String spanName = defaultTracer.getCurrentSpan().getSpan().getSpanName();
         assertEquals("span_name", spanName);
+        assertTrue(defaultTracer.isRecording());
     }
 
     @SuppressWarnings("unchecked")
diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/IntegrationTestOTelTelemetryPlugin.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/IntegrationTestOTelTelemetryPlugin.java
similarity index 85%
rename from plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/IntegrationTestOTelTelemetryPlugin.java
rename to plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/IntegrationTestOTelTelemetryPlugin.java
index ed4d13f3abb7d..45caf8bf5f60b 100644
--- a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/IntegrationTestOTelTelemetryPlugin.java
+++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/IntegrationTestOTelTelemetryPlugin.java
@@ -6,12 +6,9 @@
  * compatible open source license.
  */
 
-package org.opensearch.telemetry.tracing;
+package org.opensearch.telemetry;
 
 import org.opensearch.common.settings.Settings;
-import org.opensearch.telemetry.OTelTelemetryPlugin;
-import org.opensearch.telemetry.Telemetry;
-import org.opensearch.telemetry.TelemetrySettings;
 
 import java.util.Optional;
 
diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/InMemorySingletonMetricsExporter.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/InMemorySingletonMetricsExporter.java
new file mode 100644
index 0000000000000..74fc872cb30e3
--- /dev/null
+++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/InMemorySingletonMetricsExporter.java
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.metrics;
+
+import java.util.Collection;
+import java.util.List;
+
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.metrics.InstrumentType;
+import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
+import io.opentelemetry.sdk.metrics.data.MetricData;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
+import io.opentelemetry.sdk.testing.exporter.InMemoryMetricExporter;
+
+public class InMemorySingletonMetricsExporter implements MetricExporter {
+
+    public static final InMemorySingletonMetricsExporter INSTANCE = new InMemorySingletonMetricsExporter(InMemoryMetricExporter.create());
+
+    private static InMemoryMetricExporter delegate;
+
+    public static InMemorySingletonMetricsExporter create() {
+        return INSTANCE;
+    }
+
+    private InMemorySingletonMetricsExporter(InMemoryMetricExporter delegate) {
+        InMemorySingletonMetricsExporter.delegate = delegate;
+    }
+
+    @Override
+    public CompletableResultCode export(Collection metrics) {
+        return delegate.export(metrics);
+    }
+
+    @Override
+    public CompletableResultCode flush() {
+        return delegate.flush();
+    }
+
+    @Override
+    public CompletableResultCode shutdown() {
+        return delegate.shutdown();
+    }
+
+    public List getFinishedMetricItems() {
+        return delegate.getFinishedMetricItems();
+    }
+
+    /**
+     * Clears the state.
+     */
+    public void reset() {
+        delegate.reset();
+    }
+
+    @Override
+    public AggregationTemporality getAggregationTemporality(InstrumentType instrumentType) {
+        return delegate.getAggregationTemporality(instrumentType);
+    }
+}
diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsDisabledSanityIT.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsDisabledSanityIT.java
new file mode 100644
index 0000000000000..bcdcb657c4f42
--- /dev/null
+++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsDisabledSanityIT.java
@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.metrics;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.telemetry.IntegrationTestOTelTelemetryPlugin;
+import org.opensearch.telemetry.OTelTelemetrySettings;
+import org.opensearch.telemetry.TelemetrySettings;
+import org.opensearch.telemetry.metrics.noop.NoopCounter;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, minNumDataNodes = 1)
+public class TelemetryMetricsDisabledSanityIT extends OpenSearchIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING.getKey(), false)
+            .put(
+                OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.getKey(),
+                "org.opensearch.telemetry.metrics.InMemorySingletonMetricsExporter"
+            )
+            .put(TelemetrySettings.METRICS_PUBLISH_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(1))
+            .build();
+    }
+
+    @Override
+    protected Collection> nodePlugins() {
+        return Arrays.asList(IntegrationTestOTelTelemetryPlugin.class);
+    }
+
+    @Override
+    protected boolean addMockTelemetryPlugin() {
+        return false;
+    }
+
+    public void testSanityChecksWhenMetricsDisabled() throws Exception {
+        MetricsRegistry metricsRegistry = internalCluster().getInstance(MetricsRegistry.class);
+
+        Counter counter = metricsRegistry.createCounter("test-counter", "test", "1");
+        counter.add(1.0);
+
+        Thread.sleep(2000);
+
+        assertTrue(metricsRegistry instanceof NoopMetricsRegistry);
+        assertTrue(counter instanceof NoopCounter);
+    }
+
+}
diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsEnabledSanityIT.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsEnabledSanityIT.java
new file mode 100644
index 0000000000000..ed341595d327d
--- /dev/null
+++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/metrics/TelemetryMetricsEnabledSanityIT.java
@@ -0,0 +1,99 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.metrics;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.telemetry.IntegrationTestOTelTelemetryPlugin;
+import org.opensearch.telemetry.OTelTelemetrySettings;
+import org.opensearch.telemetry.TelemetrySettings;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.junit.After;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+import io.opentelemetry.sdk.metrics.data.DoublePointData;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, minNumDataNodes = 1)
+public class TelemetryMetricsEnabledSanityIT extends OpenSearchIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING.getKey(), true)
+            .put(
+                OTelTelemetrySettings.OTEL_METRICS_EXPORTER_CLASS_SETTING.getKey(),
+                "org.opensearch.telemetry.metrics.InMemorySingletonMetricsExporter"
+            )
+            .put(TelemetrySettings.METRICS_PUBLISH_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(1))
+            .build();
+    }
+
+    @Override
+    protected Collection> nodePlugins() {
+        return Arrays.asList(IntegrationTestOTelTelemetryPlugin.class);
+    }
+
+    @Override
+    protected boolean addMockTelemetryPlugin() {
+        return false;
+    }
+
+    public void testCounter() throws Exception {
+        MetricsRegistry metricsRegistry = internalCluster().getInstance(MetricsRegistry.class);
+        InMemorySingletonMetricsExporter.INSTANCE.reset();
+
+        Counter counter = metricsRegistry.createCounter("test-counter", "test", "1");
+        counter.add(1.0);
+        // Sleep for about 2s to wait for metrics to be published.
+        Thread.sleep(2000);
+
+        InMemorySingletonMetricsExporter exporter = InMemorySingletonMetricsExporter.INSTANCE;
+        double value = ((DoublePointData) ((ArrayList) exporter.getFinishedMetricItems()
+            .stream()
+            .filter(a -> a.getName().equals("test-counter"))
+            .collect(Collectors.toList())
+            .get(0)
+            .getDoubleSumData()
+            .getPoints()).get(0)).getValue();
+        assertEquals(1.0, value, 0.0);
+    }
+
+    public void testUpDownCounter() throws Exception {
+
+        MetricsRegistry metricsRegistry = internalCluster().getInstance(MetricsRegistry.class);
+        InMemorySingletonMetricsExporter.INSTANCE.reset();
+
+        Counter counter = metricsRegistry.createUpDownCounter("test-up-down-counter", "test", "1");
+        counter.add(1.0);
+        counter.add(-2.0);
+        // Sleep for about 2s to wait for metrics to be published.
+        Thread.sleep(2000);
+
+        InMemorySingletonMetricsExporter exporter = InMemorySingletonMetricsExporter.INSTANCE;
+        double value = ((DoublePointData) ((ArrayList) exporter.getFinishedMetricItems()
+            .stream()
+            .filter(a -> a.getName().equals("test-up-down-counter"))
+            .collect(Collectors.toList())
+            .get(0)
+            .getDoubleSumData()
+            .getPoints()).get(0)).getValue();
+        assertEquals(-1.0, value, 0.0);
+    }
+
+    @After
+    public void reset() {
+        InMemorySingletonMetricsExporter.INSTANCE.reset();
+    }
+}
diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java
index 949a58f6cab41..45ed140e1be94 100644
--- a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java
+++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerDisabledSanityIT.java
@@ -12,6 +12,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.telemetry.IntegrationTestOTelTelemetryPlugin;
 import org.opensearch.telemetry.OTelTelemetrySettings;
 import org.opensearch.telemetry.TelemetrySettings;
 import org.opensearch.test.OpenSearchIntegTestCase;
diff --git a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java
index 8a49a0abf5512..f07f2b308e801 100644
--- a/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java
+++ b/plugins/telemetry-otel/src/internalClusterTest/java/org/opensearch/telemetry/tracing/TelemetryTracerEnabledSanityIT.java
@@ -12,6 +12,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.telemetry.IntegrationTestOTelTelemetryPlugin;
 import org.opensearch.telemetry.OTelTelemetrySettings;
 import org.opensearch.telemetry.TelemetrySettings;
 import org.opensearch.telemetry.tracing.attributes.Attributes;
@@ -88,9 +89,7 @@ public void testSanityChecksWhenTracingEnabled() throws Exception {
         );
 
         InMemorySingletonSpanExporter exporter = InMemorySingletonSpanExporter.INSTANCE;
-        if (!exporter.getFinishedSpanItems().isEmpty()) {
-            validators.validate(exporter.getFinishedSpanItems(), 6);
-        }
+        validators.validate(exporter.getFinishedSpanItems(), 6);
     }
 
     private static void updateTelemetrySetting(Client client, boolean value) {
diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java
index b57876c9310f3..297ae8873636f 100644
--- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java
+++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/OTelTelemetryPlugin.java
@@ -8,14 +8,13 @@
 
 package org.opensearch.telemetry;
 
+import org.opensearch.common.concurrent.RefCountedReleasable;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.TelemetryPlugin;
-import org.opensearch.telemetry.metrics.OTelMetricsTelemetry;
 import org.opensearch.telemetry.tracing.OTelResourceProvider;
 import org.opensearch.telemetry.tracing.OTelTelemetry;
-import org.opensearch.telemetry.tracing.OTelTracingTelemetry;
 
 import java.util.Arrays;
 import java.util.List;
@@ -37,6 +36,8 @@ public class OTelTelemetryPlugin extends Plugin implements TelemetryPlugin {
 
     private final Settings settings;
 
+    private RefCountedReleasable refCountedOpenTelemetry;
+
     /**
      * Creates Otel plugin
      * @param settings cluster settings
@@ -58,20 +59,32 @@ public List> getSettings() {
 
     @Override
     public Optional getTelemetry(TelemetrySettings telemetrySettings) {
+        initializeOpenTelemetrySdk(telemetrySettings);
         return Optional.of(telemetry(telemetrySettings));
     }
 
+    private void initializeOpenTelemetrySdk(TelemetrySettings telemetrySettings) {
+        if (refCountedOpenTelemetry != null) {
+            return;
+        }
+        OpenTelemetrySdk openTelemetrySdk = OTelResourceProvider.get(telemetrySettings, settings);
+        refCountedOpenTelemetry = new RefCountedReleasable<>("openTelemetry", openTelemetrySdk, openTelemetrySdk::close);
+    }
+
     @Override
     public String getName() {
         return OTEL_TRACER_NAME;
     }
 
     private Telemetry telemetry(TelemetrySettings telemetrySettings) {
-        final OpenTelemetrySdk openTelemetry = OTelResourceProvider.get(telemetrySettings, settings);
-        return new OTelTelemetry(
-            new OTelTracingTelemetry<>(openTelemetry, openTelemetry.getSdkTracerProvider()),
-            new OTelMetricsTelemetry<>(openTelemetry.getSdkMeterProvider())
-        );
+        return new OTelTelemetry(refCountedOpenTelemetry);
+    }
+
+    @Override
+    public void close() {
+        if (refCountedOpenTelemetry != null) {
+            refCountedOpenTelemetry.close();
+        }
     }
 
 }
diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java
index 8598e5976d20d..6160e5106c041 100644
--- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java
+++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetry.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.telemetry.metrics;
 
+import org.opensearch.common.concurrent.RefCountedReleasable;
 import org.opensearch.telemetry.OTelTelemetryPlugin;
 
 import java.io.Closeable;
@@ -19,19 +20,24 @@
 import io.opentelemetry.api.metrics.DoubleUpDownCounter;
 import io.opentelemetry.api.metrics.Meter;
 import io.opentelemetry.api.metrics.MeterProvider;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
 
 /**
  * OTel implementation for {@link MetricsTelemetry}
  */
 public class OTelMetricsTelemetry implements MetricsTelemetry {
+    private final RefCountedReleasable refCountedOpenTelemetry;
     private final Meter otelMeter;
     private final T meterProvider;
 
     /**
      * Creates OTel based {@link MetricsTelemetry}.
+     * @param openTelemetry open telemetry.
      * @param meterProvider {@link MeterProvider} instance
      */
-    public OTelMetricsTelemetry(T meterProvider) {
+    public OTelMetricsTelemetry(RefCountedReleasable openTelemetry, T meterProvider) {
+        this.refCountedOpenTelemetry = openTelemetry;
+        this.refCountedOpenTelemetry.incRef();
         this.meterProvider = meterProvider;
         this.otelMeter = meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME);
     }
@@ -63,5 +69,6 @@ public Counter createUpDownCounter(String name, String description, String unit)
     @Override
     public void close() throws IOException {
         meterProvider.close();
+        refCountedOpenTelemetry.close();
     }
 }
diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java
index 282fabd43346b..0c697d2cc5e8c 100644
--- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java
+++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTelemetry.java
@@ -8,34 +8,39 @@
 
 package org.opensearch.telemetry.tracing;
 
+import org.opensearch.common.concurrent.RefCountedReleasable;
 import org.opensearch.telemetry.Telemetry;
 import org.opensearch.telemetry.metrics.MetricsTelemetry;
+import org.opensearch.telemetry.metrics.OTelMetricsTelemetry;
+
+import io.opentelemetry.sdk.OpenTelemetrySdk;
 
 /**
  * Otel implementation of Telemetry
  */
 public class OTelTelemetry implements Telemetry {
 
-    private final TracingTelemetry tracingTelemetry;
-    private final MetricsTelemetry metricsTelemetry;
+    private final RefCountedReleasable refCountedOpenTelemetry;
 
     /**
      * Creates Telemetry instance
-     * @param tracingTelemetry tracing telemetry
-     * @param metricsTelemetry metrics telemetry
+
+     */
+    /**
+     * Creates Telemetry instance
+     * @param refCountedOpenTelemetry open telemetry.
      */
-    public OTelTelemetry(TracingTelemetry tracingTelemetry, MetricsTelemetry metricsTelemetry) {
-        this.tracingTelemetry = tracingTelemetry;
-        this.metricsTelemetry = metricsTelemetry;
+    public OTelTelemetry(RefCountedReleasable refCountedOpenTelemetry) {
+        this.refCountedOpenTelemetry = refCountedOpenTelemetry;
     }
 
     @Override
     public TracingTelemetry getTracingTelemetry() {
-        return tracingTelemetry;
+        return new OTelTracingTelemetry<>(refCountedOpenTelemetry, refCountedOpenTelemetry.get().getSdkTracerProvider());
     }
 
     @Override
     public MetricsTelemetry getMetricsTelemetry() {
-        return metricsTelemetry;
+        return new OTelMetricsTelemetry<>(refCountedOpenTelemetry, refCountedOpenTelemetry.get().getSdkMeterProvider());
     }
 }
diff --git a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java
index f88afe623fd56..af39617a8c744 100644
--- a/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java
+++ b/plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelTracingTelemetry.java
@@ -8,31 +8,33 @@
 
 package org.opensearch.telemetry.tracing;
 
+import org.opensearch.common.concurrent.RefCountedReleasable;
 import org.opensearch.telemetry.OTelAttributesConverter;
 import org.opensearch.telemetry.OTelTelemetryPlugin;
 
 import java.io.Closeable;
 import java.io.IOException;
 
-import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.trace.TracerProvider;
 import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
 
 /**
  * OTel based Telemetry provider
  */
 public class OTelTracingTelemetry implements TracingTelemetry {
-    private final OpenTelemetry openTelemetry;
+    private final RefCountedReleasable refCountedOpenTelemetry;
     private final T tracerProvider;
     private final io.opentelemetry.api.trace.Tracer otelTracer;
 
     /**
      * Creates OTel based {@link TracingTelemetry}
-     * @param openTelemetry OpenTelemetry instance
+     * @param refCountedOpenTelemetry OpenTelemetry instance
      * @param tracerProvider {@link TracerProvider} instance.
      */
-    public OTelTracingTelemetry(OpenTelemetry openTelemetry, T tracerProvider) {
-        this.openTelemetry = openTelemetry;
+    public OTelTracingTelemetry(RefCountedReleasable refCountedOpenTelemetry, T tracerProvider) {
+        this.refCountedOpenTelemetry = refCountedOpenTelemetry;
+        this.refCountedOpenTelemetry.incRef();
         this.tracerProvider = tracerProvider;
         this.otelTracer = tracerProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME);
     }
@@ -40,6 +42,7 @@ public OTelTracingTelemetry(OpenTelemetry openTelemetry, T tracerProvider) {
     @Override
     public void close() throws IOException {
         tracerProvider.close();
+        refCountedOpenTelemetry.close();
     }
 
     @Override
@@ -49,7 +52,7 @@ public Span createSpan(SpanCreationContext spanCreationContext, Span parentSpan)
 
     @Override
     public TracingContextPropagator getContextPropagator() {
-        return new OTelTracingContextPropagator(openTelemetry);
+        return new OTelTracingContextPropagator(refCountedOpenTelemetry.get());
     }
 
     private Span createOtelSpan(SpanCreationContext spanCreationContext, Span parentSpan) {
diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java
index 233c93e6b9a36..9de575b69774a 100644
--- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java
+++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/metrics/OTelMetricsTelemetryTests.java
@@ -8,11 +8,13 @@
 
 package org.opensearch.telemetry.metrics;
 
+import org.opensearch.common.concurrent.RefCountedReleasable;
 import org.opensearch.telemetry.OTelAttributesConverter;
 import org.opensearch.telemetry.OTelTelemetryPlugin;
 import org.opensearch.telemetry.metrics.tags.Tags;
 import org.opensearch.test.OpenSearchTestCase;
 
+import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.metrics.DoubleCounter;
 import io.opentelemetry.api.metrics.DoubleCounterBuilder;
 import io.opentelemetry.api.metrics.DoubleUpDownCounter;
@@ -34,12 +36,16 @@ public void testCounter() {
         String description = "test";
         String unit = "1";
         Meter mockMeter = mock(Meter.class);
+        OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class);
         DoubleCounter mockOTelDoubleCounter = mock(DoubleCounter.class);
         LongCounterBuilder mockOTelLongCounterBuilder = mock(LongCounterBuilder.class);
         DoubleCounterBuilder mockOTelDoubleCounterBuilder = mock(DoubleCounterBuilder.class);
         MeterProvider meterProvider = mock(MeterProvider.class);
         when(meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockMeter);
-        MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(meterProvider);
+        MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(
+            new RefCountedReleasable("telemetry", mockOpenTelemetry, () -> {}),
+            meterProvider
+        );
         when(mockMeter.counterBuilder(counterName)).thenReturn(mockOTelLongCounterBuilder);
         when(mockOTelLongCounterBuilder.setDescription(description)).thenReturn(mockOTelLongCounterBuilder);
         when(mockOTelLongCounterBuilder.setUnit(unit)).thenReturn(mockOTelLongCounterBuilder);
@@ -59,6 +65,7 @@ public void testCounterNegativeValue() {
         String counterName = "test-counter";
         String description = "test";
         String unit = "1";
+        OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class);
         Meter mockMeter = mock(Meter.class);
         DoubleCounter mockOTelDoubleCounter = mock(DoubleCounter.class);
         LongCounterBuilder mockOTelLongCounterBuilder = mock(LongCounterBuilder.class);
@@ -66,7 +73,10 @@ public void testCounterNegativeValue() {
 
         MeterProvider meterProvider = mock(MeterProvider.class);
         when(meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockMeter);
-        MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(meterProvider);
+        MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(
+            new RefCountedReleasable("telemetry", mockOpenTelemetry, () -> {}),
+            meterProvider
+        );
         when(mockMeter.counterBuilder(counterName)).thenReturn(mockOTelLongCounterBuilder);
         when(mockOTelLongCounterBuilder.setDescription(description)).thenReturn(mockOTelLongCounterBuilder);
         when(mockOTelLongCounterBuilder.setUnit(unit)).thenReturn(mockOTelLongCounterBuilder);
@@ -83,6 +93,7 @@ public void testUpDownCounter() {
         String counterName = "test-counter";
         String description = "test";
         String unit = "1";
+        OpenTelemetry mockOpenTelemetry = mock(OpenTelemetry.class);
         Meter mockMeter = mock(Meter.class);
         DoubleUpDownCounter mockOTelUpDownDoubleCounter = mock(DoubleUpDownCounter.class);
         LongUpDownCounterBuilder mockOTelLongUpDownCounterBuilder = mock(LongUpDownCounterBuilder.class);
@@ -90,7 +101,10 @@ public void testUpDownCounter() {
 
         MeterProvider meterProvider = mock(MeterProvider.class);
         when(meterProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockMeter);
-        MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(meterProvider);
+        MetricsTelemetry metricsTelemetry = new OTelMetricsTelemetry(
+            new RefCountedReleasable("telemetry", mockOpenTelemetry, () -> {}),
+            meterProvider
+        );
         when(mockMeter.upDownCounterBuilder(counterName)).thenReturn(mockOTelLongUpDownCounterBuilder);
         when(mockOTelLongUpDownCounterBuilder.setDescription(description)).thenReturn(mockOTelLongUpDownCounterBuilder);
         when(mockOTelLongUpDownCounterBuilder.setUnit(unit)).thenReturn(mockOTelLongUpDownCounterBuilder);
diff --git a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java
index 1a508ed252493..1f0c2f674e655 100644
--- a/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java
+++ b/plugins/telemetry-otel/src/test/java/org/opensearch/telemetry/tracing/OTelTracingTelemetryTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.telemetry.tracing;
 
+import org.opensearch.common.concurrent.RefCountedReleasable;
 import org.opensearch.telemetry.OTelTelemetryPlugin;
 import org.opensearch.telemetry.tracing.attributes.Attributes;
 import org.opensearch.test.OpenSearchTestCase;
@@ -37,7 +38,10 @@ public void testCreateSpanWithoutParent() {
         when(mockSpanBuilder.startSpan()).thenReturn(mock(io.opentelemetry.api.trace.Span.class));
         when(mockSpanBuilder.setSpanKind(any(io.opentelemetry.api.trace.SpanKind.class))).thenReturn(mockSpanBuilder);
         Attributes attributes = Attributes.create().addAttribute("name", "value");
-        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider);
+        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(
+            new RefCountedReleasable("telemetry", mockOpenTelemetry, () -> {}),
+            mockTracerProvider
+        );
         Span span = tracingTelemetry.createSpan(SpanCreationContext.internal().name("span_name").attributes(attributes), null);
         verify(mockSpanBuilder, never()).setParent(any());
         verify(mockSpanBuilder).setAllAttributes(createAttribute(attributes));
@@ -59,7 +63,10 @@ public void testCreateSpanWithParent() {
 
         Span parentSpan = new OTelSpan("parent_span", mock(io.opentelemetry.api.trace.Span.class), null);
 
-        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider);
+        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(
+            new RefCountedReleasable("telemetry", mockOpenTelemetry, () -> {}),
+            mockTracerProvider
+        );
         Attributes attributes = Attributes.create().addAttribute("name", 1l);
         Span span = tracingTelemetry.createSpan(SpanCreationContext.internal().name("span_name").attributes(attributes), parentSpan);
 
@@ -85,7 +92,10 @@ public void testCreateSpanWithParentWithMultipleAttributes() {
 
         Span parentSpan = new OTelSpan("parent_span", mock(io.opentelemetry.api.trace.Span.class), null);
 
-        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider);
+        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(
+            new RefCountedReleasable("telemetry", mockOpenTelemetry, () -> {}),
+            mockTracerProvider
+        );
         Attributes attributes = Attributes.create()
             .addAttribute("key1", 1l)
             .addAttribute("key2", 2.0)
@@ -125,7 +135,10 @@ public void testGetContextPropagator() {
         TracerProvider mockTracerProvider = mock(TracerProvider.class);
         when(mockTracerProvider.get(OTelTelemetryPlugin.INSTRUMENTATION_SCOPE_NAME)).thenReturn(mockTracer);
 
-        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(mockOpenTelemetry, mockTracerProvider);
+        TracingTelemetry tracingTelemetry = new OTelTracingTelemetry(
+            new RefCountedReleasable("telemetry", mockOpenTelemetry, () -> {}),
+            mockTracerProvider
+        );
 
         assertTrue(tracingTelemetry.getContextPropagator() instanceof OTelTracingContextPropagator);
     }
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index bb0910c4e5f9c..90f91dcb7c553 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -700,6 +700,12 @@ public void apply(Settings value, Settings current, Settings previous) {
             SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING
         ),
         List.of(FeatureFlags.TELEMETRY),
-        List.of(TelemetrySettings.TRACER_ENABLED_SETTING, TelemetrySettings.TRACER_SAMPLER_PROBABILITY)
+        List.of(
+            TelemetrySettings.TRACER_ENABLED_SETTING,
+            TelemetrySettings.TRACER_SAMPLER_PROBABILITY,
+            TelemetrySettings.METRICS_PUBLISH_INTERVAL_SETTING,
+            TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING,
+            TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING
+        )
     );
 }
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index c456f01135dee..69b80462bbf0b 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -600,10 +600,23 @@ protected Node(
             MetricsRegistryFactory metricsRegistryFactory;
             if (FeatureFlags.isEnabled(TELEMETRY)) {
                 final TelemetrySettings telemetrySettings = new TelemetrySettings(settings, clusterService.getClusterSettings());
-                List telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class);
-                TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, telemetrySettings);
-                tracerFactory = new TracerFactory(telemetrySettings, telemetryModule.getTelemetry(), threadPool.getThreadContext());
-                metricsRegistryFactory = new MetricsRegistryFactory(telemetrySettings, telemetryModule.getTelemetry());
+                if (telemetrySettings.isTracingFeatureEnabled() || telemetrySettings.isMetricsFeatureEnabled()) {
+                    List telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class);
+                    TelemetryModule telemetryModule = new TelemetryModule(telemetryPlugins, telemetrySettings);
+                    if (telemetrySettings.isTracingFeatureEnabled()) {
+                        tracerFactory = new TracerFactory(telemetrySettings, telemetryModule.getTelemetry(), threadPool.getThreadContext());
+                    } else {
+                        tracerFactory = new NoopTracerFactory();
+                    }
+                    if (telemetrySettings.isMetricsFeatureEnabled()) {
+                        metricsRegistryFactory = new MetricsRegistryFactory(telemetrySettings, telemetryModule.getTelemetry());
+                    } else {
+                        metricsRegistryFactory = new NoopMetricsRegistryFactory();
+                    }
+                } else {
+                    tracerFactory = new NoopTracerFactory();
+                    metricsRegistryFactory = new NoopMetricsRegistryFactory();
+                }
             } else {
                 tracerFactory = new NoopTracerFactory();
                 metricsRegistryFactory = new NoopMetricsRegistryFactory();
diff --git a/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java b/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java
index edb20cfa9dfc5..24dcab98c8870 100644
--- a/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java
+++ b/server/src/main/java/org/opensearch/telemetry/TelemetrySettings.java
@@ -28,6 +28,20 @@ public class TelemetrySettings {
         Setting.Property.Dynamic
     );
 
+    public static final Setting TRACER_FEATURE_ENABLED_SETTING = Setting.boolSetting(
+        "telemetry.feature.tracer.enabled",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Final
+    );
+
+    public static final Setting METRICS_FEATURE_ENABLED_SETTING = Setting.boolSetting(
+        "telemetry.feature.metrics.enabled",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Final
+    );
+
     /**
      * Probability of sampler
      */
@@ -53,9 +67,14 @@ public class TelemetrySettings {
     private volatile boolean tracingEnabled;
     private volatile double samplingProbability;
 
+    private final boolean tracingFeatureEnabled;
+    private final boolean metricsFeatureEnabled;
+
     public TelemetrySettings(Settings settings, ClusterSettings clusterSettings) {
         this.tracingEnabled = TRACER_ENABLED_SETTING.get(settings);
         this.samplingProbability = TRACER_SAMPLER_PROBABILITY.get(settings);
+        this.tracingFeatureEnabled = TRACER_FEATURE_ENABLED_SETTING.get(settings);
+        this.metricsFeatureEnabled = METRICS_FEATURE_ENABLED_SETTING.get(settings);
 
         clusterSettings.addSettingsUpdateConsumer(TRACER_ENABLED_SETTING, this::setTracingEnabled);
         clusterSettings.addSettingsUpdateConsumer(TRACER_SAMPLER_PROBABILITY, this::setSamplingProbability);
@@ -83,4 +102,12 @@ public void setSamplingProbability(double samplingProbability) {
     public double getSamplingProbability() {
         return samplingProbability;
     }
+
+    public boolean isTracingFeatureEnabled() {
+        return tracingFeatureEnabled;
+    }
+
+    public boolean isMetricsFeatureEnabled() {
+        return metricsFeatureEnabled;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java
index b2308402379ac..631fb8242d78e 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/WrappedTracer.java
@@ -59,6 +59,11 @@ public SpanScope withSpanInScope(Span span) {
         return getDelegateTracer().withSpanInScope(span);
     }
 
+    @Override
+    public boolean isRecording() {
+        return getDelegateTracer().isRecording();
+    }
+
     @Override
     public void close() throws IOException {
         defaultTracer.close();
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java
index 0a9757310fe8b..e0fb690bd29be 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableHttpChannel.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.telemetry.tracing.channels;
 
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.http.HttpChannel;
 import org.opensearch.http.HttpResponse;
@@ -50,7 +49,7 @@ private TraceableHttpChannel(HttpChannel delegate, Span span, Tracer tracer) {
      * @return http channel
      */
     public static HttpChannel create(HttpChannel delegate, Span span, Tracer tracer) {
-        if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) {
+        if (tracer.isRecording() == true) {
             return new TraceableHttpChannel(delegate, span, tracer);
         } else {
             return delegate;
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java
index d256c9d4d0e53..32769dd1d848d 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableRestChannel.java
@@ -9,7 +9,6 @@
 package org.opensearch.telemetry.tracing.channels;
 
 import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.rest.RestChannel;
@@ -53,7 +52,7 @@ private TraceableRestChannel(RestChannel delegate, Span span, Tracer tracer) {
      * @return rest channel
      */
     public static RestChannel create(RestChannel delegate, Span span, Tracer tracer) {
-        if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) {
+        if (tracer.isRecording() == true) {
             return new TraceableRestChannel(delegate, span, tracer);
         } else {
             return delegate;
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java
index bd60c35c3baac..45268b4807cd9 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/channels/TraceableTcpTransportChannel.java
@@ -9,7 +9,6 @@
 package org.opensearch.telemetry.tracing.channels;
 
 import org.opensearch.Version;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.telemetry.tracing.Span;
@@ -53,7 +52,7 @@ public TraceableTcpTransportChannel(TcpTransportChannel delegate, Span span, Tra
      * @return transport channel
      */
     public static TransportChannel create(TcpTransportChannel delegate, final Span span, final Tracer tracer) {
-        if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) {
+        if (tracer.isRecording() == true) {
             delegate.getChannel().addCloseListener(new ActionListener() {
                 @Override
                 public void onResponse(Void unused) {
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java
index 538bf82a1dbec..eb9d53d2df51b 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.telemetry.tracing.handler;
 
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.telemetry.tracing.Span;
@@ -55,7 +54,7 @@ public static  TransportResponseHandler create(
         Span span,
         Tracer tracer
     ) {
-        if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) {
+        if (tracer.isRecording() == true) {
             return new TraceableTransportResponseHandler(delegate, span, tracer);
         } else {
             return delegate;
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java b/server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java
index 3e201641a529b..0cb4ce71d05f8 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/listener/TraceableActionListener.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.telemetry.tracing.listener;
 
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.telemetry.tracing.Span;
 import org.opensearch.telemetry.tracing.SpanScope;
@@ -47,7 +46,7 @@ private TraceableActionListener(ActionListener delegate, Span span, Tr
      * @return action listener
      */
     public static  ActionListener create(ActionListener delegate, Span span, Tracer tracer) {
-        if (FeatureFlags.isEnabled(FeatureFlags.TELEMETRY) == true) {
+        if (tracer.isRecording() == true) {
             return new TraceableActionListener(delegate, span, tracer);
         } else {
             return delegate;
diff --git a/server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java b/server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java
index 5d5ea62dd161e..80942123fd4fd 100644
--- a/server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java
+++ b/server/src/test/java/org/opensearch/telemetry/metrics/MetricsRegistryFactoryTests.java
@@ -63,6 +63,18 @@ public void testGetMetricsWithAvailableMetricsTelemetry() {
 
     }
 
+    public void testNullMetricsTelemetry() {
+        Settings settings = Settings.builder().put(TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING.getKey(), false).build();
+        TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings()));
+        Telemetry mockTelemetry = mock(Telemetry.class);
+        when(mockTelemetry.getMetricsTelemetry()).thenReturn(null);
+        metricsRegistryFactory = new MetricsRegistryFactory(telemetrySettings, Optional.of(mockTelemetry));
+
+        MetricsRegistry metricsRegistry = metricsRegistryFactory.getMetricsRegistry();
+        assertTrue(metricsRegistry instanceof NoopMetricsRegistry);
+
+    }
+
     private Set> getClusterSettings() {
         Set> allTracerSettings = new HashSet<>();
         ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add));
diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java
index b27f888eaf502..3a388be22445e 100644
--- a/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java
+++ b/server/src/test/java/org/opensearch/telemetry/tracing/TracerFactoryTests.java
@@ -83,6 +83,18 @@ public void testGetTracerWithAvailableTracingTelemetryReturnsWrappedTracer() {
 
     }
 
+    public void testNullTracer() {
+        Settings settings = Settings.builder().put(TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING.getKey(), false).build();
+        TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings()));
+        Telemetry mockTelemetry = mock(Telemetry.class);
+        when(mockTelemetry.getTracingTelemetry()).thenReturn(null);
+        tracerFactory = new TracerFactory(telemetrySettings, Optional.of(mockTelemetry), new ThreadContext(Settings.EMPTY));
+
+        Tracer tracer = tracerFactory.getTracer();
+        assertTrue(tracer instanceof NoopTracer);
+
+    }
+
     private Set> getClusterSettings() {
         Set> allTracerSettings = new HashSet<>();
         ClusterSettings.FEATURE_FLAGGED_CLUSTER_SETTINGS.get(List.of(FeatureFlags.TELEMETRY)).stream().forEach((allTracerSettings::add));
diff --git a/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java b/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java
index 43e0cb8e44439..8606104d26103 100644
--- a/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java
+++ b/server/src/test/java/org/opensearch/telemetry/tracing/WrappedTracerTests.java
@@ -26,6 +26,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class WrappedTracerTests extends OpenSearchTestCase {
 
@@ -38,6 +39,7 @@ public void testStartSpanWithTracingDisabledInvokesNoopTracer() throws Exception
             SpanCreationContext spanCreationContext = SpanCreationContext.internal().name("foo");
             wrappedTracer.startSpan(spanCreationContext);
             assertTrue(wrappedTracer.getDelegateTracer() instanceof NoopTracer);
+            assertFalse(wrappedTracer.isRecording());
             verify(mockDefaultTracer, never()).startSpan(SpanCreationContext.internal().name("foo"));
         }
     }
@@ -46,12 +48,13 @@ public void testStartSpanWithTracingEnabledInvokesDefaultTracer() throws Excepti
         Settings settings = Settings.builder().put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true).build();
         TelemetrySettings telemetrySettings = new TelemetrySettings(settings, new ClusterSettings(settings, getClusterSettings()));
         DefaultTracer mockDefaultTracer = mock(DefaultTracer.class);
-
+        when(mockDefaultTracer.isRecording()).thenReturn(true);
         try (WrappedTracer wrappedTracer = new WrappedTracer(telemetrySettings, mockDefaultTracer)) {
             SpanCreationContext spanCreationContext = SpanCreationContext.internal().name("foo");
             wrappedTracer.startSpan(spanCreationContext);
 
             assertTrue(wrappedTracer.getDelegateTracer() instanceof DefaultTracer);
+            assertTrue(wrappedTracer.isRecording());
             verify(mockDefaultTracer).startSpan(eq(spanCreationContext));
         }
     }
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index da829a3bc5225..c16cc1d2a5fba 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -1928,6 +1928,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
 
         // Enable tracer only when Telemetry Setting is enabled
         if (featureFlagSettings().getAsBoolean(FeatureFlags.TELEMETRY_SETTING.getKey(), false)) {
+            builder.put(TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING.getKey(), true);
             builder.put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true);
         }
         if (FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING.get(featureFlagSettings)) {
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java
index f14fe3bf3961c..efc29d1c254e6 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java
@@ -254,6 +254,7 @@ private Node newNode() {
             .putList(INITIAL_CLUSTER_MANAGER_NODES_SETTING.getKey(), nodeName)
             .put(FeatureFlags.TELEMETRY_SETTING.getKey(), true)
             .put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true)
+            .put(TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING.getKey(), true)
             .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .put(featureFlagSettings);
         if (FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING.get(featureFlagSettings)) {
@@ -271,6 +272,7 @@ private Node newNode() {
             plugins.add(MockHttpTransport.TestPlugin.class);
         }
         plugins.add(MockScriptService.TestPlugin.class);
+
         plugins.add(MockTelemetryPlugin.class);
         Node node = new MockNode(settingsBuilder.build(), plugins, forbidPrivateIndexSettings());
         try {
diff --git a/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java
index 95321a7009be9..dda413ce2818e 100644
--- a/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java
+++ b/test/framework/src/main/java/org/opensearch/test/telemetry/MockTelemetry.java
@@ -12,6 +12,7 @@
 import org.opensearch.telemetry.TelemetrySettings;
 import org.opensearch.telemetry.metrics.Counter;
 import org.opensearch.telemetry.metrics.MetricsTelemetry;
+import org.opensearch.telemetry.metrics.noop.NoopCounter;
 import org.opensearch.telemetry.tracing.TracingTelemetry;
 import org.opensearch.test.telemetry.tracing.MockTracingTelemetry;
 
@@ -37,12 +38,12 @@ public MetricsTelemetry getMetricsTelemetry() {
         return new MetricsTelemetry() {
             @Override
             public Counter createCounter(String name, String description, String unit) {
-                return null;
+                return NoopCounter.INSTANCE;
             }
 
             @Override
             public Counter createUpDownCounter(String name, String description, String unit) {
-                return null;
+                return NoopCounter.INSTANCE;
             }
 
             @Override

From 8b04521ffb8b5be312d1814a208560d53f4c8ae7 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Tue, 17 Oct 2023 16:00:35 -0700
Subject: [PATCH 173/388] Fix flaky test
 testFlushThrowsFlushFailedExceptionOnCorruption. (#10671)

This test is flaky in two ways, one it may hit a case where a merge occurs and the hardcoded segment does not exist to delete.
Two with certain mock filesystems a corruption exception won't be thrown here, only a NoSuchFile - This doesn't change our
handling in the engine, but the test should conditionally handle if the store is marked corrupted on close.

Signed-off-by: Marc Handalian 
---
 .../index/engine/NRTReplicationEngineTests.java   | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java
index 09484cd1b5840..57509c5daa2b1 100644
--- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java
@@ -617,11 +617,18 @@ public void testFlushThrowsFlushFailedExceptionOnCorruption() throws Exception {
         indexOperations(nrtEngine, operations);
         // wipe the nrt directory initially so we can sync with primary.
         cleanAndCopySegmentsFromPrimary(nrtEngine);
-        nrtEngineStore.directory().deleteFile("_0.si");
+        final Optional toDelete = Set.of(nrtEngineStore.directory().listAll()).stream().filter(f -> f.endsWith(".si")).findAny();
+        assertTrue(toDelete.isPresent());
+        nrtEngineStore.directory().deleteFile(toDelete.get());
         assertThrows(FlushFailedEngineException.class, nrtEngine::flush);
-        assertTrue(nrtEngineStore.isMarkedCorrupted());
-        // store will throw when eventually closed, not handled here.
-        assertThrows(RuntimeException.class, nrtEngineStore::close);
+        nrtEngine.close();
+        if (nrtEngineStore.isMarkedCorrupted()) {
+            assertThrows(RuntimeException.class, nrtEngineStore::close);
+        } else {
+            // With certain mock directories a NoSuchFileException is thrown which is not treated as a
+            // corruption Exception. In these cases we don't expect any issue on store close.
+            nrtEngineStore.close();
+        }
     }
 
     private void copySegments(Collection latestPrimaryFiles, Engine nrtEngine) throws IOException {

From 9cc0e7db3b5dfaa9b3dd111eba9d1fb4d9185549 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Wed, 18 Oct 2023 07:26:36 +0530
Subject: [PATCH 174/388] Change filenames for IndexMetadata and Manifest
 (#10557)

* Change filenames for IndexMetadata and Manifest

Signed-off-by: Dhwanil Patel 
---
 CHANGELOG.md                                  |  1 +
 .../remote/RemoteClusterStateService.java     | 38 ++++++++++++-------
 .../RemoteClusterStateServiceTests.java       | 38 +++++++++++++++++++
 3 files changed, 63 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e379002d254ae..3471564dba15e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
 - [Admission control] Add enhancements to FS stats to include read/write time, queue size and IO time ([#10541](https://github.com/opensearch-project/OpenSearch/pull/10541))
 - [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
+- Change file names for remote cluster state ([#10557](https://github.com/opensearch-project/OpenSearch/pull/10557))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 4a8a0618ffa60..0cf97de53d5f3 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -120,6 +120,9 @@ public class RemoteClusterStateService implements Closeable {
 
     private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false);
 
+    public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1;
+    public static final int MANIFEST_CURRENT_CODEC_VERSION = 1;
+
     public RemoteClusterStateService(
         String nodeId,
         Supplier repositoriesService,
@@ -426,7 +429,7 @@ private ClusterMetadataManifest uploadManifest(
         boolean committed
     ) throws IOException {
         synchronized (this) {
-            final String manifestFileName = getManifestFileName(clusterState.term(), clusterState.version());
+            final String manifestFileName = getManifestFileName(clusterState.term(), clusterState.version(), committed);
             final ClusterMetadataManifest manifest = new ClusterMetadataManifest(
                 clusterState.term(),
                 clusterState.getVersion(),
@@ -488,22 +491,30 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) {
         this.slowWriteLoggingThreshold = slowWriteLoggingThreshold;
     }
 
-    private static String getManifestFileName(long term, long version) {
-        // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest_2147483642_2147483637_456536447
-        return String.join(DELIMITER, getManifestFileNamePrefix(term, version), RemoteStoreUtils.invertLong(System.currentTimeMillis()));
-    }
-
-    private static String getManifestFileNamePrefix(long term, long version) {
-        // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest_2147483642_2147483637
-        return String.join(DELIMITER, MANIFEST_PATH_TOKEN, RemoteStoreUtils.invertLong(term), RemoteStoreUtils.invertLong(version));
+    static String getManifestFileName(long term, long version, boolean committed) {
+        // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____
+        return String.join(
+            DELIMITER,
+            MANIFEST_PATH_TOKEN,
+            RemoteStoreUtils.invertLong(term),
+            RemoteStoreUtils.invertLong(version),
+            (committed ? "C" : "P"), // C for committed and P for published
+            RemoteStoreUtils.invertLong(System.currentTimeMillis()),
+            String.valueOf(MANIFEST_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last place to
+                                                           // determine codec version.
+        );
     }
 
-    private static String indexMetadataFileName(IndexMetadata indexMetadata) {
+    static String indexMetadataFileName(IndexMetadata indexMetadata) {
+        // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index//metadata______
         return String.join(
             DELIMITER,
             INDEX_METADATA_FILE_PREFIX,
-            String.valueOf(indexMetadata.getVersion()),
-            String.valueOf(System.currentTimeMillis())
+            RemoteStoreUtils.invertLong(indexMetadata.getVersion()),
+            RemoteStoreUtils.invertLong(System.currentTimeMillis()),
+            String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last
+                                                                 // place to determine codec version.
         );
     }
 
@@ -916,8 +927,7 @@ private void deleteClusterMetadata(
                     if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) {
                         staleIndexMetadataPaths.add(
                             new BlobPath().add(INDEX_PATH_TOKEN).add(uploadedIndexMetadata.getIndexUUID()).buildAsString()
-                                + uploadedIndexMetadata.getUploadedFilename()
-                                + ".dat"
+                                + INDEX_METADATA_FORMAT.blobName(uploadedIndexMetadata.getUploadedFilename())
                         );
                     }
                 });
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 6ecbc23f75bee..119d19cc34981 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -34,6 +34,7 @@
 import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
+import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.repositories.FilterRepository;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.RepositoryMissingException;
@@ -65,6 +66,9 @@
 import org.mockito.ArgumentMatchers;
 
 import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_FILE_PREFIX;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
@@ -673,6 +677,40 @@ public void testDeleteStaleClusterUUIDs() throws IOException {
         }
     }
 
+    public void testFileNames() {
+        final Index index = new Index("test-index", "index-uuid");
+        final Settings idxSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
+            .build();
+        final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+
+        String indexMetadataFileName = RemoteClusterStateService.indexMetadataFileName(indexMetadata);
+        String[] splittedIndexMetadataFileName = indexMetadataFileName.split(DELIMITER);
+        assertThat(indexMetadataFileName.split(DELIMITER).length, is(4));
+        assertThat(splittedIndexMetadataFileName[0], is(INDEX_METADATA_FILE_PREFIX));
+        assertThat(splittedIndexMetadataFileName[1], is(RemoteStoreUtils.invertLong(indexMetadata.getVersion())));
+        assertThat(splittedIndexMetadataFileName[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION)));
+
+        int term = randomIntBetween(5, 10);
+        int version = randomIntBetween(5, 10);
+        String manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, true);
+        assertThat(manifestFileName.split(DELIMITER).length, is(6));
+        String[] splittedName = manifestFileName.split(DELIMITER);
+        assertThat(splittedName[0], is(MANIFEST_FILE_PREFIX));
+        assertThat(splittedName[1], is(RemoteStoreUtils.invertLong(term)));
+        assertThat(splittedName[2], is(RemoteStoreUtils.invertLong(version)));
+        assertThat(splittedName[3], is("C"));
+        assertThat(splittedName[5], is(String.valueOf(MANIFEST_CURRENT_CODEC_VERSION)));
+
+        manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, false);
+        splittedName = manifestFileName.split(DELIMITER);
+        assertThat(splittedName[3], is("P"));
+    }
+
     private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException {
         final BlobPath blobPath = mock(BlobPath.class);
         when((blobStoreRepository.basePath())).thenReturn(blobPath);

From 8276cbdf0749c4bcb62a53312ea6009a830f1e41 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Wed, 18 Oct 2023 17:11:06 +0530
Subject: [PATCH 175/388] revive remote cluster state auto restore integ tests
 (#10503)

* revive remote cluster state auto restore integ tests

Signed-off-by: bansvaru 
---
 .../RemoteStoreBaseIntegTestCase.java         |   8 +-
 .../RemoteStoreClusterStateRestoreIT.java     | 231 ++++++------------
 .../opensearch/test/InternalTestCluster.java  |  10 +-
 3 files changed, 82 insertions(+), 167 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
index e2ef5f85abc74..bccca283ba772 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
@@ -353,7 +353,13 @@ public void assertRemoteStoreRepositoryOnAllNodes(String repositoryName) {
             // Validated that all the restricted settings are entact on all the nodes.
             repository.getRestrictedSystemRepositorySettings()
                 .stream()
-                .forEach(setting -> assertEquals(setting.get(actualRepository.settings()), setting.get(expectedRepository.settings())));
+                .forEach(
+                    setting -> assertEquals(
+                        String.format(Locale.ROOT, "Restricted Settings mismatch [%s]", setting.getKey()),
+                        setting.get(actualRepository.settings()),
+                        setting.get(expectedRepository.settings())
+                    )
+                );
         }
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index 5e92bb195680b..3df4cc4e34d93 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -8,23 +8,22 @@
 
 package org.opensearch.remotestore;
 
-import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse;
-import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
-import org.opensearch.action.support.PlainActionFuture;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.gateway.remote.ClusterMetadataManifest;
+import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
 import org.opensearch.gateway.remote.RemoteClusterStateService;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.io.IOException;
 import java.nio.file.Files;
-import java.util.Locale;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.ExecutionException;
 
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
-import static org.opensearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE;
-import static org.opensearch.indices.ShardLimitValidator.SETTING_MAX_SHARDS_PER_CLUSTER_KEY;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreClusterStateRestoreIT extends BaseRemoteStoreRestoreIT {
@@ -48,47 +47,10 @@ private Map initialTestSetup(int shardCount, int replicaCount, int
 
     private void resetCluster(int dataNodeCount, int clusterManagerNodeCount) {
         internalCluster().stopAllNodes();
-        addNewNodes(dataNodeCount, clusterManagerNodeCount);
+        internalCluster().startClusterManagerOnlyNodes(clusterManagerNodeCount);
+        internalCluster().startDataOnlyNodes(dataNodeCount);
     }
 
-    private void restoreAndValidate(String clusterUUID, Map indexStats) throws Exception {
-        restoreAndValidate(clusterUUID, indexStats, true);
-    }
-
-    private void restoreAndValidate(String clusterUUID, Map indexStats, boolean validate) throws Exception {
-        // TODO once auto restore is merged, the remote cluster state will be restored
-
-        if (validate) {
-            // Step - 4 validation restore is successful.
-            ensureGreen(INDEX_NAME);
-            verifyRestoredData(indexStats, INDEX_NAME);
-        }
-    }
-
-    private void restoreAndValidateFails(
-        String clusterUUID,
-        PlainActionFuture actionListener,
-        Class clazz,
-        String errorSubString
-    ) {
-
-        try {
-            restoreAndValidate(clusterUUID, null, false);
-        } catch (Exception e) {
-            assertTrue(
-                String.format(Locale.ROOT, "%s %s", clazz, e),
-                clazz.isAssignableFrom(e.getClass())
-                    || clazz.isAssignableFrom(e.getCause().getClass())
-                    || (e.getCause().getCause() != null && clazz.isAssignableFrom(e.getCause().getCause().getClass()))
-            );
-            assertTrue(
-                String.format(Locale.ROOT, "Error message mismatch. Expected: [%s]. Actual: [%s]", errorSubString, e.getMessage()),
-                e.getMessage().contains(errorSubString)
-            );
-        }
-    }
-
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9834")
     public void testFullClusterRestore() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
@@ -106,10 +68,10 @@ public void testFullClusterRestore() throws Exception {
         assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
 
         // Step - 3 Trigger full cluster restore and validate
-        restoreAndValidate(prevClusterUUID, indexStats);
+        validateMetadata(List.of(INDEX_NAME));
+        verifyRestoredData(indexStats, INDEX_NAME);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9834")
     public void testFullClusterRestoreMultipleIndices() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
@@ -134,155 +96,100 @@ public void testFullClusterRestoreMultipleIndices() throws Exception {
         assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
 
         // Step - 3 Trigger full cluster restore
-        restoreAndValidate(prevClusterUUID, indexStats);
-        ensureGreen(secondIndexName);
-        verifyRestoredData(indexStats2, secondIndexName);
+        validateMetadata(List.of(INDEX_NAME, secondIndexName));
+        verifyRestoredData(indexStats, INDEX_NAME);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9834")
-    public void testFullClusterRestoreFailureValidationFailures() throws Exception {
+    public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathThrowsException() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
         int dataNodeCount = shardCount * (replicaCount + 1);
         int clusterManagerNodeCount = 1;
 
-        // index some data to generate files in remote directory
-        Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount);
-        String prevClusterUUID = clusterService().state().metadata().clusterUUID();
-
-        // Start of Test - 1
-        // Test - 1 Trigger full cluster restore and validate it fails due to incorrect cluster UUID
-        PlainActionFuture future = PlainActionFuture.newFuture();
-        restoreAndValidateFails("randomUUID", future, IllegalStateException.class, "Remote Cluster State not found - randomUUID");
-        // End of Test - 1
+        // Step - 1 index some data to generate files in remote directory
+        initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount);
 
-        // Start of Test - 3
-        // Test - 2 Trigger full cluster restore and validate it fails due to cluster UUID same as current cluster UUID
-        future = PlainActionFuture.newFuture();
-        restoreAndValidateFails(
-            clusterService().state().metadata().clusterUUID(),
-            future,
-            IllegalArgumentException.class,
-            "clusterUUID to restore from should be different from current cluster UUID"
-        );
-        // End of Test - 2
+        String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+        String clusterName = clusterService().state().getClusterName().value();
 
-        // Start of Test - 3
         // Step - 2 Replace all nodes in the cluster with new nodes. This ensures new cluster state doesn't have previous index metadata
-        // Restarting cluster with just 1 data node helps with applying cluster settings
-        resetCluster(1, clusterManagerNodeCount);
-        String newClusterUUID = clusterService().state().metadata().clusterUUID();
-        assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
-
-        reduceShardLimits(1, 1);
-
-        // Step - 4 Trigger full cluster restore and validate it fails
-        future = PlainActionFuture.newFuture();
-        restoreAndValidateFails(
-            prevClusterUUID,
-            future,
-            IllegalArgumentException.class,
-            "this action would add [2] total shards, but this cluster currently has [0]/[1] maximum shards open"
-        );
-        resetShardLimits();
-        // End of Test - 3
-
-        // Start of Test - 4
-        // Test -4 Reset cluster and trigger full restore with same name index in the cluster
-        // Test -4 Add required nodes for this test after last reset.
-        addNewNodes(dataNodeCount - 1, 0);
-
-        newClusterUUID = clusterService().state().metadata().clusterUUID();
-        assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
-
-        // Test -4 Step - 2 Create a new index with same name
-        createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 1));
-        ensureYellowAndNoInitializingShards(INDEX_NAME);
-        ensureGreen(INDEX_NAME);
-
-        future = PlainActionFuture.newFuture();
-
-        // Test -4 Step - 3 Trigger full cluster restore and validate fails
-        restoreAndValidateFails(
-            prevClusterUUID,
-            future,
-            IllegalStateException.class,
-            "cannot restore index [remote-store-test-idx-1] because an open index with same name/uuid already exists in the cluster"
-        );
+        internalCluster().stopAllNodes();
+        // Step - 3 Delete index metadata file in remote
+        try {
+            Files.move(
+                segmentRepoPath.resolve(
+                    RemoteClusterStateService.encodeString(clusterName) + "/cluster-state/" + prevClusterUUID + "/index"
+                ),
+                segmentRepoPath.resolve("cluster-state/")
+            );
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        assertThrows(IllegalStateException.class, () -> addNewNodes(dataNodeCount, clusterManagerNodeCount));
+        // Test is complete
 
-        // Test -4 Step - 4 validation restore is successful.
-        ensureGreen(INDEX_NAME);
-        // End of Test - 4
+        // Starting a node without remote state to ensure test cleanup
+        internalCluster().startNode(Settings.builder().put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), false).build());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9834")
-    public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathThrowsException() throws Exception {
+    public void testRemoteStateFullRestart() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
         int dataNodeCount = shardCount * (replicaCount + 1);
-        int clusterManagerNodeCount = 1;
-
-        // Step - 1 index some data to generate files in remote directory
-        initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount);
+        int clusterManagerNodeCount = 3;
 
+        Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount);
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
-
-        // Step - 2 Replace all nodes in the cluster with new nodes. This ensures new cluster state doesn't have previous index metadata
-        resetCluster(dataNodeCount, clusterManagerNodeCount);
-
-        String newClusterUUID = clusterService().state().metadata().clusterUUID();
-        assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
-
-        // Step - 4 Delete index metadata file in remote
+        // Delete index metadata file in remote
         try {
             Files.move(
                 segmentRepoPath.resolve(
                     RemoteClusterStateService.encodeString(clusterService().state().getClusterName().value())
                         + "/cluster-state/"
                         + prevClusterUUID
-                        + "/index"
+                        + "/manifest"
                 ),
                 segmentRepoPath.resolve("cluster-state/")
             );
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-
-        // Step - 5 Trigger full cluster restore and validate fails
-        PlainActionFuture future = PlainActionFuture.newFuture();
-        restoreAndValidateFails(prevClusterUUID, future, IllegalStateException.class, "asdsa");
+        internalCluster().fullRestart();
+        ensureGreen(INDEX_NAME);
+        String newClusterUUID = clusterService().state().metadata().clusterUUID();
+        assert Objects.equals(newClusterUUID, prevClusterUUID) : "Full restart not successful. cluster uuid has changed";
+        validateCurrentMetadata();
+        verifyRestoredData(indexStats, INDEX_NAME);
     }
 
-    private void reduceShardLimits(int maxShardsPerNode, int maxShardsPerCluster) {
-        // Step 3 - Reduce shard limits to hit shard limit with less no of shards
-        try {
-            client().admin()
-                .cluster()
-                .updateSettings(
-                    new ClusterUpdateSettingsRequest().transientSettings(
-                        Settings.builder()
-                            .put(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey(), maxShardsPerNode)
-                            .put(SETTING_MAX_SHARDS_PER_CLUSTER_KEY, maxShardsPerCluster)
-                    )
-                )
-                .get();
-        } catch (InterruptedException | ExecutionException e) {
-            throw new RuntimeException(e);
+    private void validateMetadata(List indexNames) {
+        assertEquals(clusterService().state().metadata().indices().size(), indexNames.size());
+        for (String indexName : indexNames) {
+            assertTrue(clusterService().state().metadata().hasIndex(indexName));
         }
     }
 
-    private void resetShardLimits() {
-        // Step - 5 Reset the cluster settings
-        ClusterUpdateSettingsRequest resetRequest = new ClusterUpdateSettingsRequest();
-        resetRequest.transientSettings(
-            Settings.builder().putNull(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey()).putNull(SETTING_MAX_SHARDS_PER_CLUSTER_KEY)
+    private void validateCurrentMetadata() throws Exception {
+        RemoteClusterStateService remoteClusterStateService = internalCluster().getInstance(
+            RemoteClusterStateService.class,
+            internalCluster().getClusterManagerName()
         );
-
-        try {
-            client().admin().cluster().updateSettings(resetRequest).get();
-        } catch (InterruptedException | ExecutionException e) {
-            throw new RuntimeException(e);
-        }
+        assertBusy(() -> {
+            ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
+                getClusterState().getClusterName().value(),
+                getClusterState().metadata().clusterUUID()
+            ).get();
+            ClusterState clusterState = getClusterState();
+            Metadata currentMetadata = clusterState.metadata();
+            assertEquals(currentMetadata.indices().size(), manifest.getIndices().size());
+            assertEquals(currentMetadata.coordinationMetadata().term(), manifest.getClusterTerm());
+            assertEquals(clusterState.version(), manifest.getStateVersion());
+            assertEquals(clusterState.stateUUID(), manifest.getStateUUID());
+            assertEquals(currentMetadata.clusterUUIDCommitted(), manifest.isClusterUUIDCommitted());
+            for (UploadedIndexMetadata uploadedIndexMetadata : manifest.getIndices()) {
+                IndexMetadata currentIndexMetadata = currentMetadata.index(uploadedIndexMetadata.getIndexName());
+                assertEquals(currentIndexMetadata.getIndex().getUUID(), uploadedIndexMetadata.getIndexUUID());
+            }
+        });
     }
-
 }
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index 3c7423f73685f..a290f3a1751a0 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -1853,10 +1853,12 @@ public synchronized void stopRandomNodeNotCurrentMaster() throws IOException {
      */
     public void stopAllNodes() {
         try {
-            int totalDataNodes = numDataNodes();
-            while (totalDataNodes > 0) {
-                stopRandomDataNode();
-                totalDataNodes -= 1;
+            if (numDataAndClusterManagerNodes() != numClusterManagerNodes()) {
+                int totalDataNodes = numDataNodes();
+                while (totalDataNodes > 0) {
+                    stopRandomDataNode();
+                    totalDataNodes -= 1;
+                }
             }
             int totalClusterManagerNodes = numClusterManagerNodes();
             while (totalClusterManagerNodes > 1) {

From 4632f4b17e5bbd6077104d7b68f2bd28336a5922 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Wed, 18 Oct 2023 17:11:26 +0530
Subject: [PATCH 176/388] Upload global cluster state to remote store (#10404)

* Upload global cluster state to remote store

Signed-off-by: Dhwanil Patel 
---
 CHANGELOG.md                                  |   5 +-
 .../remote/ClusterMetadataManifest.java       | 128 ++++++++-
 .../remote/RemoteClusterStateService.java     | 201 +++++++++++++-
 .../blobstore/BlobStoreRepository.java        |   8 +-
 .../blobstore/ChecksumBlobStoreFormat.java    |  15 +-
 .../coordination/CoordinationStateTests.java  |   2 +
 .../remote/ClusterMetadataManifestTests.java  |  31 +++
 .../RemoteClusterStateServiceTests.java       | 260 +++++++++++++++++-
 .../snapshots/BlobStoreFormatTests.java       |   6 +-
 9 files changed, 617 insertions(+), 39 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3471564dba15e..243423223ca52 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,7 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
 - [Admission control] Add enhancements to FS stats to include read/write time, queue size and IO time ([#10541](https://github.com/opensearch-project/OpenSearch/pull/10541))
 - [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
-- Change file names for remote cluster state ([#10557](https://github.com/opensearch-project/OpenSearch/pull/10557))
+- [Remote cluster state] Change file names for remote cluster state ([#10557](https://github.com/opensearch-project/OpenSearch/pull/10557))
+- [Remote cluster state] Upload global metadata in cluster state to remote store([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -121,4 +122,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
index 40b16f3d6323b..97b37d9532f85 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
@@ -33,6 +33,9 @@
  */
 public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
 
+    public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest.
+    public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file.
+
     private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term");
     private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version");
     private static final ParseField CLUSTER_UUID_FIELD = new ParseField("cluster_uuid");
@@ -40,6 +43,8 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
     private static final ParseField OPENSEARCH_VERSION_FIELD = new ParseField("opensearch_version");
     private static final ParseField NODE_ID_FIELD = new ParseField("node_id");
     private static final ParseField COMMITTED_FIELD = new ParseField("committed");
+    private static final ParseField CODEC_VERSION_FIELD = new ParseField("codec_version");
+    private static final ParseField GLOBAL_METADATA_FIELD = new ParseField("global_metadata");
     private static final ParseField INDICES_FIELD = new ParseField("indices");
     private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid");
     private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed");
@@ -84,7 +89,33 @@ private static boolean clusterUUIDCommitted(Object[] fields) {
         return (boolean) fields[9];
     }
 
-    private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
+    private static int codecVersion(Object[] fields) {
+        return (int) fields[10];
+    }
+
+    private static String globalMetadataFileName(Object[] fields) {
+        return (String) fields[11];
+    }
+
+    private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>(
+        "cluster_metadata_manifest",
+        fields -> new ClusterMetadataManifest(
+            term(fields),
+            version(fields),
+            clusterUUID(fields),
+            stateUUID(fields),
+            opensearchVersion(fields),
+            nodeId(fields),
+            committed(fields),
+            CODEC_V0,
+            null,
+            indices(fields),
+            previousClusterUUID(fields),
+            clusterUUIDCommitted(fields)
+        )
+    );
+
+    private static final ConstructingObjectParser PARSER_V1 = new ConstructingObjectParser<>(
         "cluster_metadata_manifest",
         fields -> new ClusterMetadataManifest(
             term(fields),
@@ -94,29 +125,45 @@ private static boolean clusterUUIDCommitted(Object[] fields) {
             opensearchVersion(fields),
             nodeId(fields),
             committed(fields),
+            codecVersion(fields),
+            globalMetadataFileName(fields),
             indices(fields),
             previousClusterUUID(fields),
             clusterUUIDCommitted(fields)
         )
     );
 
+    private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V1;
+
     static {
-        PARSER.declareLong(ConstructingObjectParser.constructorArg(), CLUSTER_TERM_FIELD);
-        PARSER.declareLong(ConstructingObjectParser.constructorArg(), STATE_VERSION_FIELD);
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_FIELD);
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), STATE_UUID_FIELD);
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), OPENSEARCH_VERSION_FIELD);
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), NODE_ID_FIELD);
-        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), COMMITTED_FIELD);
-        PARSER.declareObjectArray(
+        declareParser(PARSER_V0, CODEC_V0);
+        declareParser(PARSER_V1, CODEC_V1);
+    }
+
+    private static void declareParser(ConstructingObjectParser parser, long codec_version) {
+        parser.declareLong(ConstructingObjectParser.constructorArg(), CLUSTER_TERM_FIELD);
+        parser.declareLong(ConstructingObjectParser.constructorArg(), STATE_VERSION_FIELD);
+        parser.declareString(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_FIELD);
+        parser.declareString(ConstructingObjectParser.constructorArg(), STATE_UUID_FIELD);
+        parser.declareInt(ConstructingObjectParser.constructorArg(), OPENSEARCH_VERSION_FIELD);
+        parser.declareString(ConstructingObjectParser.constructorArg(), NODE_ID_FIELD);
+        parser.declareBoolean(ConstructingObjectParser.constructorArg(), COMMITTED_FIELD);
+        parser.declareObjectArray(
             ConstructingObjectParser.constructorArg(),
             (p, c) -> UploadedIndexMetadata.fromXContent(p),
             INDICES_FIELD
         );
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
-        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_COMMITTED);
+        parser.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
+        parser.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_COMMITTED);
+
+        if (codec_version >= CODEC_V1) {
+            parser.declareInt(ConstructingObjectParser.constructorArg(), CODEC_VERSION_FIELD);
+            parser.declareString(ConstructingObjectParser.constructorArg(), GLOBAL_METADATA_FIELD);
+        }
     }
 
+    private final int codecVersion;
+    private final String globalMetadataFileName;
     private final List indices;
     private final long clusterTerm;
     private final long stateVersion;
@@ -168,6 +215,14 @@ public boolean isClusterUUIDCommitted() {
         return clusterUUIDCommitted;
     }
 
+    public int getCodecVersion() {
+        return codecVersion;
+    }
+
+    public String getGlobalMetadataFileName() {
+        return globalMetadataFileName;
+    }
+
     public ClusterMetadataManifest(
         long clusterTerm,
         long version,
@@ -176,6 +231,8 @@ public ClusterMetadataManifest(
         Version opensearchVersion,
         String nodeId,
         boolean committed,
+        int codecVersion,
+        String globalMetadataFileName,
         List indices,
         String previousClusterUUID,
         boolean clusterUUIDCommitted
@@ -187,6 +244,8 @@ public ClusterMetadataManifest(
         this.opensearchVersion = opensearchVersion;
         this.nodeId = nodeId;
         this.committed = committed;
+        this.codecVersion = codecVersion;
+        this.globalMetadataFileName = globalMetadataFileName;
         this.indices = Collections.unmodifiableList(indices);
         this.previousClusterUUID = previousClusterUUID;
         this.clusterUUIDCommitted = clusterUUIDCommitted;
@@ -203,6 +262,13 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
         this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
         this.previousClusterUUID = in.readString();
         this.clusterUUIDCommitted = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            this.codecVersion = in.readInt();
+            this.globalMetadataFileName = in.readString();
+        } else {
+            this.codecVersion = CODEC_V0; // Default codec
+            this.globalMetadataFileName = null;
+        }
     }
 
     public static Builder builder() {
@@ -231,6 +297,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.endArray();
         builder.field(PREVIOUS_CLUSTER_UUID.getPreferredName(), getPreviousClusterUUID());
         builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted());
+        if (onOrAfterCodecVersion(CODEC_V1)) {
+            builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion());
+            builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName());
+        }
         return builder;
     }
 
@@ -246,6 +316,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(indices);
         out.writeString(previousClusterUUID);
         out.writeBoolean(clusterUUIDCommitted);
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeInt(codecVersion);
+            out.writeString(globalMetadataFileName);
+        }
     }
 
     @Override
@@ -266,12 +340,16 @@ public boolean equals(Object o) {
             && Objects.equals(nodeId, that.nodeId)
             && Objects.equals(committed, that.committed)
             && Objects.equals(previousClusterUUID, that.previousClusterUUID)
-            && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted);
+            && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted)
+            && Objects.equals(globalMetadataFileName, that.globalMetadataFileName)
+            && Objects.equals(codecVersion, that.codecVersion);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(
+            codecVersion,
+            globalMetadataFileName,
             indices,
             clusterTerm,
             stateVersion,
@@ -290,8 +368,16 @@ public String toString() {
         return Strings.toString(MediaTypeRegistry.JSON, this);
     }
 
+    public boolean onOrAfterCodecVersion(int codecVersion) {
+        return this.codecVersion >= codecVersion;
+    }
+
+    public static ClusterMetadataManifest fromXContentV0(XContentParser parser) throws IOException {
+        return PARSER_V0.parse(parser, null);
+    }
+
     public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
+        return CURRENT_PARSER.parse(parser, null);
     }
 
     /**
@@ -301,6 +387,8 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws
      */
     public static class Builder {
 
+        private String globalMetadataFileName;
+        private int codecVersion;
         private List indices;
         private long clusterTerm;
         private long stateVersion;
@@ -317,6 +405,16 @@ public Builder indices(List indices) {
             return this;
         }
 
+        public Builder codecVersion(int codecVersion) {
+            this.codecVersion = codecVersion;
+            return this;
+        }
+
+        public Builder globalMetadataFileName(String globalMetadataFileName) {
+            this.globalMetadataFileName = globalMetadataFileName;
+            return this;
+        }
+
         public Builder clusterTerm(long clusterTerm) {
             this.clusterTerm = clusterTerm;
             return this;
@@ -378,6 +476,8 @@ public Builder(ClusterMetadataManifest manifest) {
             this.opensearchVersion = manifest.opensearchVersion;
             this.nodeId = manifest.nodeId;
             this.committed = manifest.committed;
+            this.globalMetadataFileName = manifest.globalMetadataFileName;
+            this.codecVersion = manifest.codecVersion;
             this.indices = new ArrayList<>(manifest.indices);
             this.previousClusterUUID = manifest.previousClusterUUID;
             this.clusterUUIDCommitted = manifest.clusterUUIDCommitted;
@@ -392,6 +492,8 @@ public ClusterMetadataManifest build() {
                 opensearchVersion,
                 nodeId,
                 committed,
+                codecVersion,
+                globalMetadataFileName,
                 indices,
                 previousClusterUUID,
                 clusterUUIDCommitted
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 0cf97de53d5f3..2092c2a0aac3f 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -15,6 +15,7 @@
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobMetadata;
@@ -27,6 +28,7 @@
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.Index;
+import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.translog.transfer.BlobStoreTransferService;
@@ -55,6 +57,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
@@ -80,7 +83,9 @@ public class RemoteClusterStateService implements Closeable {
 
     private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class);
 
+    // TODO make this two variable as dynamic setting [issue: #10688]
     public static final int INDEX_METADATA_UPLOAD_WAIT_MILLIS = 20000;
+    public static final int GLOBAL_METADATA_UPLOAD_WAIT_MILLIS = 20000;
 
     public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
         "index-metadata",
@@ -88,11 +93,27 @@ public class RemoteClusterStateService implements Closeable {
         IndexMetadata::fromXContent
     );
 
+    public static final ChecksumBlobStoreFormat GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
+        "metadata",
+        METADATA_NAME_FORMAT,
+        Metadata::fromXContent
+    );
+
+    /**
+     * Manifest format compatible with older codec v0, where codec version was missing.
+     */
+    public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V0 =
+        new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0);
+
+    /**
+     * Manifest format compatible with codec v1, where we introduced codec versions/global metadata.
+     */
     public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>(
         "cluster-metadata-manifest",
         METADATA_MANIFEST_NAME_FORMAT,
         ClusterMetadataManifest::fromXContent
     );
+
     /**
      * Used to specify if cluster state metadata should be published to remote store
      */
@@ -105,9 +126,11 @@ public class RemoteClusterStateService implements Closeable {
 
     public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state";
     public static final String INDEX_PATH_TOKEN = "index";
+    public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata";
     public static final String MANIFEST_PATH_TOKEN = "manifest";
     public static final String MANIFEST_FILE_PREFIX = "manifest";
-    public static final String INDEX_METADATA_FILE_PREFIX = "metadata";
+    public static final String METADATA_FILE_PREFIX = "metadata";
+    public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion
 
     private final String nodeId;
     private final Supplier repositoriesService;
@@ -121,7 +144,17 @@ public class RemoteClusterStateService implements Closeable {
     private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false);
 
     public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1;
-    public static final int MANIFEST_CURRENT_CODEC_VERSION = 1;
+    public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1;
+    public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1;
+
+    // ToXContent Params with gateway mode.
+    // We are using gateway context mode to persist all custom metadata.
+    public static final ToXContent.Params FORMAT_PARAMS;
+    static {
+        Map params = new HashMap<>(1);
+        params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY);
+        FORMAT_PARAMS = new ToXContent.MapParams(params);
+    }
 
     public RemoteClusterStateService(
         String nodeId,
@@ -162,12 +195,22 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri
             return null;
         }
 
+        // TODO: we can upload global metadata and index metadata in parallel. [issue: #10645]
+        // Write globalMetadata
+        String globalMetadataFile = writeGlobalMetadata(clusterState);
+
         // any validations before/after upload ?
         final List allUploadedIndexMetadata = writeIndexMetadataParallel(
             clusterState,
             new ArrayList<>(clusterState.metadata().indices().values())
         );
-        final ClusterMetadataManifest manifest = uploadManifest(clusterState, allUploadedIndexMetadata, previousClusterUUID, false);
+        final ClusterMetadataManifest manifest = uploadManifest(
+            clusterState,
+            allUploadedIndexMetadata,
+            previousClusterUUID,
+            globalMetadataFile,
+            false
+        );
         final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos);
         if (durationMillis >= slowWriteLoggingThreshold.getMillis()) {
             logger.warn(
@@ -206,6 +249,22 @@ public ClusterMetadataManifest writeIncrementalMetadata(
             return null;
         }
         assert previousClusterState.metadata().coordinationMetadata().term() == clusterState.metadata().coordinationMetadata().term();
+
+        // Write Global Metadata
+        final boolean updateGlobalMetadata = Metadata.isGlobalStateEquals(
+            previousClusterState.metadata(),
+            clusterState.metadata()
+        ) == false;
+        String globalMetadataFile;
+        // For migration case from codec V0 to V1, we have added null check on global metadata file,
+        // If file is empty and codec is 1 then write global metadata.
+        if (updateGlobalMetadata || previousManifest.getGlobalMetadataFileName() == null) {
+            globalMetadataFile = writeGlobalMetadata(clusterState);
+        } else {
+            globalMetadataFile = previousManifest.getGlobalMetadataFileName();
+        }
+
+        // Write Index Metadata
         final Map previousStateIndexMetadataVersionByName = new HashMap<>();
         for (final IndexMetadata indexMetadata : previousClusterState.metadata().indices().values()) {
             previousStateIndexMetadataVersionByName.put(indexMetadata.getIndex().getName(), indexMetadata.getVersion());
@@ -248,6 +307,7 @@ public ClusterMetadataManifest writeIncrementalMetadata(
             clusterState,
             new ArrayList<>(allUploadedIndexMetadata.values()),
             previousManifest.getPreviousClusterUUID(),
+            globalMetadataFile,
             false
         );
         deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS);
@@ -273,6 +333,59 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         return manifest;
     }
 
+    /**
+     * Uploads provided ClusterState's global Metadata to remote store in parallel.
+     * The call is blocking so the method waits for upload to finish and then return.
+     *
+     * @param clusterState current ClusterState
+     * @return String file name where globalMetadata file is stored.
+     */
+    private String writeGlobalMetadata(ClusterState clusterState) throws IOException {
+
+        AtomicReference result = new AtomicReference();
+        final BlobContainer globalMetadataContainer = globalMetadataContainer(
+            clusterState.getClusterName().value(),
+            clusterState.metadata().clusterUUID()
+        );
+        final String globalMetadataFilename = globalMetadataFileName(clusterState.metadata());
+
+        // latch to wait until upload is not finished
+        CountDownLatch latch = new CountDownLatch(1);
+
+        LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> {
+            logger.trace(String.format(Locale.ROOT, "GlobalMetadata uploaded successfully."));
+            result.set(globalMetadataContainer.path().buildAsString() + globalMetadataFilename);
+        }, ex -> { throw new GlobalMetadataTransferException(ex.getMessage(), ex); }), latch);
+
+        GLOBAL_METADATA_FORMAT.writeAsync(
+            clusterState.metadata(),
+            globalMetadataContainer,
+            globalMetadataFilename,
+            blobStoreRepository.getCompressor(),
+            completionListener,
+            FORMAT_PARAMS
+        );
+
+        try {
+            if (latch.await(GLOBAL_METADATA_UPLOAD_WAIT_MILLIS, TimeUnit.MILLISECONDS) == false) {
+                // TODO: We should add metrics where transfer is timing out. [Issue: #10687]
+                GlobalMetadataTransferException ex = new GlobalMetadataTransferException(
+                    String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete")
+                );
+                throw ex;
+            }
+        } catch (InterruptedException ex) {
+            GlobalMetadataTransferException exception = new GlobalMetadataTransferException(
+                String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete - %s"),
+                ex
+            );
+            Thread.currentThread().interrupt();
+            throw exception;
+        }
+
+        return result.get();
+    }
+
     /**
      * Uploads provided IndexMetadata's to remote store in parallel. The call is blocking so the method waits for upload to finish and then return.
      *
@@ -381,7 +494,8 @@ private void writeIndexMetadataAsync(
             indexMetadataContainer,
             indexMetadataFilename,
             blobStoreRepository.getCompressor(),
-            completionListener
+            completionListener,
+            FORMAT_PARAMS
         );
     }
 
@@ -398,6 +512,7 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat
             clusterState,
             previousManifest.getIndices(),
             previousManifest.getPreviousClusterUUID(),
+            previousManifest.getGlobalMetadataFileName(),
             true
         );
         deleteStaleClusterUUIDs(clusterState, committedManifest);
@@ -426,6 +541,7 @@ private ClusterMetadataManifest uploadManifest(
         ClusterState clusterState,
         List uploadedIndexMetadata,
         String previousClusterUUID,
+        String globalClusterMetadataFileName,
         boolean committed
     ) throws IOException {
         synchronized (this) {
@@ -438,6 +554,8 @@ private ClusterMetadataManifest uploadManifest(
                 Version.CURRENT,
                 nodeId,
                 committed,
+                MANIFEST_CURRENT_CODEC_VERSION,
+                globalClusterMetadataFileName,
                 uploadedIndexMetadata,
                 previousClusterUUID,
                 clusterState.metadata().clusterUUIDCommitted()
@@ -469,6 +587,12 @@ private BlobContainer indexMetadataContainer(String clusterName, String clusterU
             .blobContainer(getCusterMetadataBasePath(clusterName, clusterUUID).add(INDEX_PATH_TOKEN).add(indexUUID));
     }
 
+    private BlobContainer globalMetadataContainer(String clusterName, String clusterUUID) {
+        // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/
+        return blobStoreRepository.blobStore()
+            .blobContainer(getCusterMetadataBasePath(clusterName, clusterUUID).add(GLOBAL_METADATA_PATH_TOKEN));
+    }
+
     private BlobContainer manifestContainer(String clusterName, String clusterUUID) {
         // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest
         return blobStoreRepository.blobStore().blobContainer(getManifestFolderPath(clusterName, clusterUUID));
@@ -510,7 +634,7 @@ static String indexMetadataFileName(IndexMetadata indexMetadata) {
         // version>
         return String.join(
             DELIMITER,
-            INDEX_METADATA_FILE_PREFIX,
+            METADATA_FILE_PREFIX,
             RemoteStoreUtils.invertLong(indexMetadata.getVersion()),
             RemoteStoreUtils.invertLong(System.currentTimeMillis()),
             String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last
@@ -518,6 +642,17 @@ static String indexMetadataFileName(IndexMetadata indexMetadata) {
         );
     }
 
+    private static String globalMetadataFileName(Metadata metadata) {
+        // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/metadata______
+        return String.join(
+            DELIMITER,
+            METADATA_FILE_PREFIX,
+            RemoteStoreUtils.invertLong(metadata.version()),
+            RemoteStoreUtils.invertLong(System.currentTimeMillis()),
+            String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION)
+        );
+    }
+
     private BlobPath getManifestFolderPath(String clusterName, String clusterUUID) {
         return getCusterMetadataBasePath(clusterName, clusterUUID).add(MANIFEST_PATH_TOKEN);
     }
@@ -601,7 +736,8 @@ public String getLastKnownUUIDFromRemote(String clusterName) {
             return validChain.get(0);
         } catch (IOException e) {
             throw new IllegalStateException(
-                String.format(Locale.ROOT, "Error while fetching previous UUIDs from remote store for cluster name: %s", clusterName)
+                String.format(Locale.ROOT, "Error while fetching previous UUIDs from remote store for cluster name: %s", clusterName),
+                e
             );
         }
     }
@@ -622,7 +758,8 @@ private Map getLatestManifestForAllClusterUUIDs
                 manifest.ifPresent(clusterMetadataManifest -> manifestsByClusterUUID.put(clusterUUID, clusterMetadataManifest));
             } catch (Exception e) {
                 throw new IllegalStateException(
-                    String.format(Locale.ROOT, "Exception in fetching manifest for clusterUUID: %s", clusterUUID)
+                    String.format(Locale.ROOT, "Exception in fetching manifest for clusterUUID: %s", clusterUUID),
+                    e
                 );
             }
         }
@@ -788,7 +925,7 @@ private Optional getLatestManifestFileName(String clusterName, String cl
     private ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, String clusterUUID, String filename)
         throws IllegalStateException {
         try {
-            return RemoteClusterStateService.CLUSTER_METADATA_MANIFEST_FORMAT.read(
+            return getClusterMetadataManifestBlobStoreFormat(filename).read(
                 manifestContainer(clusterName, clusterUUID),
                 filename,
                 blobStoreRepository.getNamedXContentRegistry()
@@ -798,6 +935,29 @@ private ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String cluste
         }
     }
 
+    private ChecksumBlobStoreFormat getClusterMetadataManifestBlobStoreFormat(String fileName) {
+        long codecVersion = getManifestCodecVersion(fileName);
+        if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) {
+            return CLUSTER_METADATA_MANIFEST_FORMAT;
+        } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) {
+            return CLUSTER_METADATA_MANIFEST_FORMAT_V0;
+        }
+
+        throw new IllegalArgumentException("Cluster metadata manifest file is corrupted, don't have valid codec version");
+    }
+
+    private int getManifestCodecVersion(String fileName) {
+        String[] splitName = fileName.split(DELIMITER);
+        if (splitName.length == SPLITED_MANIFEST_FILE_LENGTH) {
+            return Integer.parseInt(splitName[splitName.length - 1]); // Last value would be codec version.
+        } else if (splitName.length < SPLITED_MANIFEST_FILE_LENGTH) { // Where codec is not part of file name, i.e. default codec version 0
+                                                                      // is used.
+            return ClusterMetadataManifest.CODEC_V0;
+        } else {
+            throw new IllegalArgumentException("Manifest file name is corrupted");
+        }
+    }
+
     public static String encodeString(String content) {
         return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8));
     }
@@ -816,6 +976,20 @@ public IndexMetadataTransferException(String errorDesc, Throwable cause) {
         }
     }
 
+    /**
+     * Exception for GlobalMetadata transfer failures to remote
+     */
+    static class GlobalMetadataTransferException extends RuntimeException {
+
+        public GlobalMetadataTransferException(String errorDesc) {
+            super(errorDesc);
+        }
+
+        public GlobalMetadataTransferException(String errorDesc, Throwable cause) {
+            super(errorDesc, cause);
+        }
+    }
+
     /**
      * Purges all remote cluster state against provided cluster UUIDs
      *
@@ -907,6 +1081,7 @@ private void deleteClusterMetadata(
             Set filesToKeep = new HashSet<>();
             Set staleManifestPaths = new HashSet<>();
             Set staleIndexMetadataPaths = new HashSet<>();
+            Set staleGlobalMetadataPaths = new HashSet<>();
             activeManifestBlobMetadata.forEach(blobMetadata -> {
                 ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest(
                     clusterName,
@@ -915,6 +1090,7 @@ private void deleteClusterMetadata(
                 );
                 clusterMetadataManifest.getIndices()
                     .forEach(uploadedIndexMetadata -> filesToKeep.add(uploadedIndexMetadata.getUploadedFilename()));
+                filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName());
             });
             staleManifestBlobMetadata.forEach(blobMetadata -> {
                 ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest(
@@ -923,6 +1099,14 @@ private void deleteClusterMetadata(
                     blobMetadata.name()
                 );
                 staleManifestPaths.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blobMetadata.name());
+                if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) {
+                    String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/");
+                    staleGlobalMetadataPaths.add(
+                        new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName(
+                            globalMetadataSplitPath[globalMetadataSplitPath.length - 1]
+                        )
+                    );
+                }
                 clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> {
                     if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) {
                         staleIndexMetadataPaths.add(
@@ -938,6 +1122,7 @@ private void deleteClusterMetadata(
                 return;
             }
 
+            deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleGlobalMetadataPaths));
             deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexMetadataPaths));
             deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleManifestPaths));
         } catch (IllegalStateException e) {
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index 69883e0d19c8d..c48efa9d3d2ee 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -173,6 +173,7 @@
 import java.util.stream.Stream;
 
 import static org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName;
+import static org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS;
 
 /**
  * BlobStore - based implementation of Snapshot Repository
@@ -3333,7 +3334,12 @@ private void writeShardIndexBlobAtomic(
             () -> new ParameterizedMessage("[{}] Writing shard index [{}] to [{}]", metadata.name(), indexGeneration, shardContainer.path())
         );
         final String blobName = INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(String.valueOf(indexGeneration));
-        writeAtomic(shardContainer, blobName, INDEX_SHARD_SNAPSHOTS_FORMAT.serialize(updatedSnapshots, blobName, compressor), true);
+        writeAtomic(
+            shardContainer,
+            blobName,
+            INDEX_SHARD_SNAPSHOTS_FORMAT.serialize(updatedSnapshots, blobName, compressor, SNAPSHOT_ONLY_FORMAT_PARAMS),
+            true
+        );
     }
 
     // Unused blobs are all previous index-, data- and meta-blobs and that are not referenced by the new index- as well as all
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 7e1960171043a..17cb68f798094 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -83,7 +83,7 @@
 public final class ChecksumBlobStoreFormat {
 
     // Serialization parameters to specify correct context for metadata serialization
-    private static final ToXContent.Params SNAPSHOT_ONLY_FORMAT_PARAMS;
+    public static final ToXContent.Params SNAPSHOT_ONLY_FORMAT_PARAMS;
 
     static {
         Map snapshotOnlyParams = new HashMap<>();
@@ -171,7 +171,7 @@ public T deserialize(String blobName, NamedXContentRegistry namedXContentRegistr
      */
     public void write(final T obj, final BlobContainer blobContainer, final String name, final Compressor compressor) throws IOException {
         final String blobName = blobName(name);
-        final BytesReference bytes = serialize(obj, blobName, compressor);
+        final BytesReference bytes = serialize(obj, blobName, compressor, SNAPSHOT_ONLY_FORMAT_PARAMS);
         blobContainer.writeBlob(blobName, bytes.streamInput(), bytes.length(), false);
     }
 
@@ -184,13 +184,15 @@ public void write(final T obj, final BlobContainer blobContainer, final String n
      * @param name                blob name
      * @param compressor          whether to use compression
      * @param listener            listener to listen to write result
+     * @param params              ToXContent params
      */
     public void writeAsync(
         final T obj,
         final BlobContainer blobContainer,
         final String name,
         final Compressor compressor,
-        ActionListener listener
+        ActionListener listener,
+        final ToXContent.Params params
     ) throws IOException {
         if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) {
             write(obj, blobContainer, name, compressor);
@@ -198,7 +200,7 @@ public void writeAsync(
             return;
         }
         final String blobName = blobName(name);
-        final BytesReference bytes = serialize(obj, blobName, compressor);
+        final BytesReference bytes = serialize(obj, blobName, compressor, params);
         final String resourceDescription = "ChecksumBlobStoreFormat.writeAsync(blob=\"" + blobName + "\")";
         try (IndexInput input = new ByteArrayIndexInput(resourceDescription, BytesReference.toBytes(bytes))) {
             long expectedChecksum;
@@ -230,7 +232,8 @@ public void writeAsync(
         }
     }
 
-    public BytesReference serialize(final T obj, final String blobName, final Compressor compressor) throws IOException {
+    public BytesReference serialize(final T obj, final String blobName, final Compressor compressor, final ToXContent.Params params)
+        throws IOException {
         try (BytesStreamOutput outputStream = new BytesStreamOutput()) {
             try (
                 OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(
@@ -254,7 +257,7 @@ public void close() throws IOException {
                     )
                 ) {
                     builder.startObject();
-                    obj.toXContent(builder, SNAPSHOT_ONLY_FORMAT_PARAMS);
+                    obj.toXContent(builder, params);
                     builder.endObject();
                 }
                 CodecUtil.writeFooter(indexOutput);
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java
index f37823d2c0c7d..1c0dc7fc1ca2d 100644
--- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java
+++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java
@@ -938,6 +938,8 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep
             Version.CURRENT,
             randomAlphaOfLength(10),
             false,
+            1,
+            randomAlphaOfLength(10),
             Collections.emptyList(),
             randomAlphaOfLength(10),
             true
diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
index 66426c2a880a3..6c9a3201656d7 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
@@ -26,6 +26,33 @@
 
 public class ClusterMetadataManifestTests extends OpenSearchTestCase {
 
+    public void testClusterMetadataManifestXContentV0() throws IOException {
+        UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
+        ClusterMetadataManifest originalManifest = new ClusterMetadataManifest(
+            1L,
+            1L,
+            "test-cluster-uuid",
+            "test-state-uuid",
+            Version.CURRENT,
+            "test-node-id",
+            false,
+            ClusterMetadataManifest.CODEC_V0,
+            null,
+            Collections.singletonList(uploadedIndexMetadata),
+            "prev-cluster-uuid",
+            true
+        );
+        final XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
+            final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV0(parser);
+            assertEquals(originalManifest, fromXContentManifest);
+        }
+    }
+
     public void testClusterMetadataManifestXContent() throws IOException {
         UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
         ClusterMetadataManifest originalManifest = new ClusterMetadataManifest(
@@ -36,6 +63,8 @@ public void testClusterMetadataManifestXContent() throws IOException {
             Version.CURRENT,
             "test-node-id",
             false,
+            ClusterMetadataManifest.CODEC_V1,
+            "test-global-metadata-file",
             Collections.singletonList(uploadedIndexMetadata),
             "prev-cluster-uuid",
             true
@@ -60,6 +89,8 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() {
             Version.CURRENT,
             "B10RX1f5RJenMQvYccCgSQ",
             true,
+            1,
+            "test-global-metadata-file",
             randomUploadedIndexMetadataList(),
             "yfObdx8KSMKKrXf8UyHhM",
             true
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 119d19cc34981..ddee3bcf10382 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -66,10 +66,11 @@
 import org.mockito.ArgumentMatchers;
 
 import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION;
-import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_FILE_PREFIX;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
@@ -232,14 +233,15 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
         assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName()));
         assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID()));
         assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue());
+        assertThat(manifest.getGlobalMetadataFileName(), notNullValue());
         assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm()));
         assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion()));
         assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
         assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID()));
 
-        assertEquals(actionListenerArgumentCaptor.getAllValues().size(), 1);
-        assertEquals(writeContextArgumentCaptor.getAllValues().size(), 1);
+        assertEquals(actionListenerArgumentCaptor.getAllValues().size(), 2);
+        assertEquals(writeContextArgumentCaptor.getAllValues().size(), 2);
 
         WriteContext capturedWriteContext = writeContextArgumentCaptor.getValue();
         byte[] writtenBytes = capturedWriteContext.getStreamProvider(Integer.MAX_VALUE).provideStream(0).getInputStream().readAllBytes();
@@ -263,7 +265,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
 
     }
 
-    public void testWriteFullMetadataInParallelFailure() throws IOException {
+    public void testWriteFullMetadataFailureForGlobalMetadata() throws IOException {
         final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
         AsyncMultiStreamBlobContainer container = (AsyncMultiStreamBlobContainer) mockBlobStoreObjects(AsyncMultiStreamBlobContainer.class);
 
@@ -274,6 +276,27 @@ public void testWriteFullMetadataInParallelFailure() throws IOException {
             return null;
         }).when(container).asyncBlobUpload(any(WriteContext.class), actionListenerArgumentCaptor.capture());
 
+        remoteClusterStateService.start();
+        assertThrows(
+            RemoteClusterStateService.GlobalMetadataTransferException.class,
+            () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10))
+        );
+    }
+
+    public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOException {
+        final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        AsyncMultiStreamBlobContainer container = (AsyncMultiStreamBlobContainer) mockBlobStoreObjects(AsyncMultiStreamBlobContainer.class);
+
+        ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
+
+        doAnswer((i) -> {
+            actionListenerArgumentCaptor.getValue().onResponse(null);
+            return null;
+        }).doAnswer((i) -> {
+            actionListenerArgumentCaptor.getValue().onFailure(new RuntimeException("Cannot upload to remote"));
+            return null;
+        }).when(container).asyncBlobUpload(any(WriteContext.class), actionListenerArgumentCaptor.capture());
+
         remoteClusterStateService.start();
         assertThrows(
             RemoteClusterStateService.IndexMetadataTransferException.class,
@@ -338,6 +361,207 @@ public void testWriteIncrementalMetadataSuccess() throws IOException {
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
     }
 
+    /*
+     * Here we will verify the migration of manifest file from codec V0 and V1.
+     *
+     * Initially codec version is 0 and global metadata is also null, we will perform index metadata update.
+     * In final manifest codec version should be 1 and
+     * global metadata should be updated, even if it was not changed in this cluster state update
+     */
+    public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOException {
+        mockBlobStoreObjects();
+        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
+        final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
+            .nodes(nodesWithLocalNodeClusterManager())
+            .build();
+
+        // Update only index metadata
+        final IndexMetadata indexMetadata = new IndexMetadata.Builder("test").settings(
+            Settings.builder()
+                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
+                .build()
+        ).numberOfShards(1).numberOfReplicas(0).build();
+        Metadata newMetadata = Metadata.builder(previousClusterState.metadata()).put(indexMetadata, true).build();
+        ClusterState newClusterState = ClusterState.builder(previousClusterState).metadata(newMetadata).build();
+
+        // previous manifest with codec 0 and null global metadata
+        final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
+            .codecVersion(ClusterMetadataManifest.CODEC_V0)
+            .globalMetadataFileName(null)
+            .indices(Collections.emptyList())
+            .build();
+
+        remoteClusterStateService.start();
+        final ClusterMetadataManifest manifestAfterUpdate = remoteClusterStateService.writeIncrementalMetadata(
+            previousClusterState,
+            newClusterState,
+            previousManifest
+        );
+
+        // global metadata is updated
+        assertThat(manifestAfterUpdate.getGlobalMetadataFileName(), notNullValue());
+        // Manifest file with codec version with 1 is updated.
+        assertThat(manifestAfterUpdate.getCodecVersion(), is(ClusterMetadataManifest.CODEC_V1));
+    }
+
+    public void testWriteIncrementalGlobalMetadataSuccess() throws IOException {
+        final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build();
+        mockBlobStoreObjects();
+        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
+        final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
+            .build();
+
+        final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder()
+            .codecVersion(2)
+            .globalMetadataFileName("global-metadata-file")
+            .indices(Collections.emptyList())
+            .build();
+
+        remoteClusterStateService.start();
+        final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata(
+            previousClusterState,
+            clusterState,
+            previousManifest
+        );
+
+        final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
+            .indices(Collections.emptyList())
+            .globalMetadataFileName("mock-filename")
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .stateUUID("state-uuid")
+            .clusterUUID("cluster-uuid")
+            .previousClusterUUID("prev-cluster-uuid")
+            .build();
+
+        assertThat(manifest.getGlobalMetadataFileName(), notNullValue());
+        assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm()));
+        assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion()));
+        assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID()));
+        assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
+    }
+
+    /*
+     * Here we will verify index metadata is not uploaded again if change is only in global metadata
+     */
+    public void testGlobalMetadataOnlyUpdated() throws IOException {
+        // setup
+        mockBlobStoreObjects();
+        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
+        final ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
+            .build();
+        final ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder()
+            .codecVersion(2)
+            .globalMetadataFileName("global-metadata-file")
+            .indices(Collections.emptyList())
+            .build();
+        remoteClusterStateService.start();
+
+        // Initial cluster state with index.
+        final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        // Updating remote cluster state with changing index metadata
+        final ClusterMetadataManifest manifestAfterIndexMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
+            initialClusterState,
+            clusterState,
+            initialManifest
+        );
+
+        // new cluster state where only global metadata is different
+        Metadata newMetadata = Metadata.builder(clusterState.metadata())
+            .persistentSettings(Settings.builder().put("cluster.blocks.read_only", true).build())
+            .build();
+        ClusterState newClusterState = ClusterState.builder(clusterState).metadata(newMetadata).build();
+
+        // updating remote cluster state with global metadata
+        final ClusterMetadataManifest manifestAfterGlobalMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
+            clusterState,
+            newClusterState,
+            manifestAfterIndexMetadataUpdate
+        );
+
+        // Verify that index metadata information is same in manifest files
+        assertThat(manifestAfterIndexMetadataUpdate.getIndices().size(), is(manifestAfterGlobalMetadataUpdate.getIndices().size()));
+        assertThat(
+            manifestAfterIndexMetadataUpdate.getIndices().get(0).getIndexName(),
+            is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getIndexName())
+        );
+        assertThat(
+            manifestAfterIndexMetadataUpdate.getIndices().get(0).getIndexUUID(),
+            is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getIndexUUID())
+        );
+
+        // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata update
+        assertThat(
+            manifestAfterIndexMetadataUpdate.getIndices().get(0).getUploadedFilename(),
+            is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getUploadedFilename())
+        );
+
+        // global metadata file would have changed
+        assertFalse(
+            manifestAfterIndexMetadataUpdate.getGlobalMetadataFileName()
+                .equalsIgnoreCase(manifestAfterGlobalMetadataUpdate.getGlobalMetadataFileName())
+        );
+    }
+
+    /*
+     * Here we will verify global metadata is not uploaded again if change is only in index metadata
+     */
+    public void testIndexMetadataOnlyUpdated() throws IOException {
+        // setup
+        mockBlobStoreObjects();
+        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
+        final ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
+            .build();
+        final ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder()
+            .codecVersion(2)
+            .indices(Collections.emptyList())
+            .build();
+        remoteClusterStateService.start();
+
+        // Initial cluster state with global metadata.
+        final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build();
+
+        // Updating remote cluster state with changing global metadata
+        final ClusterMetadataManifest manifestAfterGlobalMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
+            initialClusterState,
+            clusterState,
+            initialManifest
+        );
+
+        // new cluster state where only Index metadata is different
+        final IndexMetadata indexMetadata = new IndexMetadata.Builder("test").settings(
+            Settings.builder()
+                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
+                .build()
+        ).numberOfShards(1).numberOfReplicas(0).build();
+        Metadata newMetadata = Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build();
+        ClusterState newClusterState = ClusterState.builder(clusterState).metadata(newMetadata).build();
+
+        // updating remote cluster state with index metadata
+        final ClusterMetadataManifest manifestAfterIndexMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata(
+            clusterState,
+            newClusterState,
+            manifestAfterGlobalMetadataUpdate
+        );
+
+        // Verify that global metadata information is same in manifest files after updating index Metadata
+        // since timestamp is part of file name, if file name is same we can confirm that file is not update in index metadata update
+        assertThat(
+            manifestAfterIndexMetadataUpdate.getGlobalMetadataFileName(),
+            is(manifestAfterGlobalMetadataUpdate.getGlobalMetadataFileName())
+        );
+
+        // Index metadata would have changed
+        assertThat(manifestAfterGlobalMetadataUpdate.getIndices().size(), is(0));
+        assertThat(manifestAfterIndexMetadataUpdate.getIndices().size(), is(1));
+    }
+
     public void testReadLatestMetadataManifestFailedIOException() throws IOException {
         final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
 
@@ -402,6 +626,7 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
+            .codecVersion(ClusterMetadataManifest.CODEC_V0)
             .build();
 
         BlobContainer blobContainer = mockBlobStoreObjects();
@@ -458,6 +683,7 @@ public void testReadLatestMetadataManifestSuccess() throws IOException {
             .clusterUUID("cluster-uuid")
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
+            .codecVersion(ClusterMetadataManifest.CODEC_V0)
             .previousClusterUUID("prev-cluster-uuid")
             .build();
 
@@ -504,6 +730,7 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
+            .codecVersion(ClusterMetadataManifest.CODEC_V0)
             .build();
 
         mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
@@ -691,7 +918,7 @@ public void testFileNames() {
         String indexMetadataFileName = RemoteClusterStateService.indexMetadataFileName(indexMetadata);
         String[] splittedIndexMetadataFileName = indexMetadataFileName.split(DELIMITER);
         assertThat(indexMetadataFileName.split(DELIMITER).length, is(4));
-        assertThat(splittedIndexMetadataFileName[0], is(INDEX_METADATA_FILE_PREFIX));
+        assertThat(splittedIndexMetadataFileName[0], is(METADATA_FILE_PREFIX));
         assertThat(splittedIndexMetadataFileName[1], is(RemoteStoreUtils.invertLong(indexMetadata.getVersion())));
         assertThat(splittedIndexMetadataFileName[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION)));
 
@@ -820,6 +1047,7 @@ private ClusterMetadataManifest generateClusterMetadataManifest(
             .previousClusterUUID(previousClusterUUID)
             .committed(true)
             .clusterUUIDCommitted(true)
+            .globalMetadataFileName("test-global-metadata")
             .build();
     }
 
@@ -859,7 +1087,8 @@ private void mockBlobContainer(
         BytesReference bytes = RemoteClusterStateService.CLUSTER_METADATA_MANIFEST_FORMAT.serialize(
             clusterMetadataManifest,
             "manifestFileName",
-            blobStoreRepository.getCompressor()
+            blobStoreRepository.getCompressor(),
+            FORMAT_PARAMS
         );
         when(blobContainer.readBlob("manifestFileName")).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
 
@@ -873,7 +1102,8 @@ private void mockBlobContainer(
                 BytesReference bytesIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.serialize(
                     indexMetadata,
                     fileName,
-                    blobStoreRepository.getCompressor()
+                    blobStoreRepository.getCompressor(),
+                    FORMAT_PARAMS
                 );
                 when(blobContainer.readBlob(fileName + ".dat")).thenReturn(
                     new ByteArrayInputStream(bytesIndexMetadata.streamInput().readAllBytes())
@@ -884,6 +1114,22 @@ private void mockBlobContainer(
         });
     }
 
+    private static ClusterState.Builder generateClusterStateWithGlobalMetadata() {
+        final Settings clusterSettings = Settings.builder().put("cluster.blocks.read_only", true).build();
+        final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
+
+        return ClusterState.builder(ClusterName.DEFAULT)
+            .version(1L)
+            .stateUUID("state-uuid")
+            .metadata(
+                Metadata.builder()
+                    .persistentSettings(clusterSettings)
+                    .clusterUUID("cluster-uuid")
+                    .coordinationMetadata(coordinationMetadata)
+                    .build()
+            );
+    }
+
     private static ClusterState.Builder generateClusterStateWithOneIndex() {
         final Index index = new Index("test-index", "index-uuid");
         final Settings idxSettings = Settings.builder()
diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
index 03f0d27188027..c114b56bd0b39 100644
--- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
@@ -152,14 +152,16 @@ public void onFailure(Exception e) {
             mockBlobContainer,
             "check-smile",
             CompressorRegistry.none(),
-            actionListener
+            actionListener,
+            ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS
         );
         checksumSMILE.writeAsync(
             new BlobObj("checksum smile compressed"),
             mockBlobContainer,
             "check-smile-comp",
             CompressorRegistry.getCompressor(DeflateCompressor.NAME),
-            actionListener
+            actionListener,
+            ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS
         );
 
         latch.await();

From e942483e716255b8f801dc1242f909d73019aa70 Mon Sep 17 00:00:00 2001
From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com>
Date: Wed, 18 Oct 2023 04:45:49 -0700
Subject: [PATCH 177/388] [Segment Replication] Unmute Test
 SegmentReplicationTargetServiceTests.testShardAlreadyReplicating (#10660)

* Remove Awaits fix.

Signed-off-by: Rishikesh1159 

* apply spotlesscheck.

Signed-off-by: Rishikesh1159 

* add test logging.

Signed-off-by: Rishikesh1159 

---------

Signed-off-by: Rishikesh1159 
---
 .../replication/SegmentReplicationTargetServiceTests.java      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
index c108de5ee5ea6..d16a188f405e3 100644
--- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
@@ -39,6 +39,7 @@
 import org.opensearch.indices.replication.common.ReplicationLuceneIndex;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
+import org.opensearch.test.junit.annotations.TestLogging;
 import org.opensearch.test.transport.CapturingTransport;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -246,7 +247,7 @@ public void testAlreadyOnNewCheckpoint() {
         verify(spy, times(0)).startReplication(any(), any(), any());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8928")
+    @TestLogging(reason = "Getting trace logs from replication package", value = "org.opensearch.indices.replication:TRACE")
     public void testShardAlreadyReplicating() {
         CountDownLatch blockGetCheckpointMetadata = new CountDownLatch(1);
         SegmentReplicationSource source = new TestReplicationSource() {

From 75bd9f2d341efbe0a795c267af923eb48bffbd91 Mon Sep 17 00:00:00 2001
From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com>
Date: Wed, 18 Oct 2023 18:14:55 +0530
Subject: [PATCH 178/388] Added BufferedInputStream to allow mark and reset ops
 during IO errors (#10690)

Signed-off-by: vikasvb90 
---
 .../repositories/s3/S3RepositoryPlugin.java        | 14 ++++++++++----
 .../repositories/s3/async/AsyncPartsHandler.java   |  6 +++++-
 .../s3/async/AsyncTransferManager.java             |  5 ++++-
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
index a80ee0ca35fae..c6450e49d08e2 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
@@ -38,6 +38,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
@@ -55,6 +56,7 @@
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.ExecutorBuilder;
 import org.opensearch.threadpool.FixedExecutorBuilder;
+import org.opensearch.threadpool.ScalingExecutorBuilder;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.watcher.ResourceWatcherService;
 
@@ -93,17 +95,21 @@ public S3RepositoryPlugin(final Settings settings, final Path configPath) {
     @Override
     public List> getExecutorBuilders(Settings settings) {
         List> executorBuilders = new ArrayList<>();
+        int halfProcMaxAt5 = halfAllocatedProcessorsMaxFive(allocatedProcessors(settings));
         executorBuilders.add(
             new FixedExecutorBuilder(settings, PRIORITY_FUTURE_COMPLETION, priorityPoolCount(settings), 10_000, PRIORITY_FUTURE_COMPLETION)
         );
-        executorBuilders.add(
-            new FixedExecutorBuilder(settings, PRIORITY_STREAM_READER, priorityPoolCount(settings), 10_000, PRIORITY_STREAM_READER)
-        );
+        executorBuilders.add(new ScalingExecutorBuilder(PRIORITY_STREAM_READER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
+
         executorBuilders.add(new FixedExecutorBuilder(settings, FUTURE_COMPLETION, normalPoolCount(settings), 10_000, FUTURE_COMPLETION));
-        executorBuilders.add(new FixedExecutorBuilder(settings, STREAM_READER, normalPoolCount(settings), 10_000, STREAM_READER));
+        executorBuilders.add(new ScalingExecutorBuilder(STREAM_READER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
         return executorBuilders;
     }
 
+    static int halfAllocatedProcessorsMaxFive(final int allocatedProcessors) {
+        return boundedBy((allocatedProcessors + 1) / 2, 1, 5);
+    }
+
     S3RepositoryPlugin(final Settings settings, final Path configPath, final S3Service service, final S3AsyncService s3AsyncService) {
         this.service = Objects.requireNonNull(service, "S3 service must not be null");
         this.configPath = configPath;
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
index ad6939ce299d6..86bb70e5a40a2 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
@@ -23,9 +23,11 @@
 import org.opensearch.common.StreamContext;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.common.io.InputStreamContainer;
+import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.s3.SocketAccess;
 import org.opensearch.repositories.s3.io.CheckedContainer;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -142,7 +144,9 @@ private static void uploadPart(
             () -> s3AsyncClient.uploadPart(
                 uploadPartRequest,
                 AsyncRequestBody.fromInputStream(
-                    inputStreamContainer.getInputStream(),
+                    // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
+                    // data can be retried instead of retrying whole file by the application.
+                    new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1)),
                     inputStreamContainer.getContentLength(),
                     streamReadExecutor
                 )
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
index 8d45c2167a3d1..fb1e409388a69 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
@@ -37,6 +37,7 @@
 import org.opensearch.repositories.s3.SocketAccess;
 import org.opensearch.repositories.s3.io.CheckedContainer;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Base64;
@@ -303,7 +304,9 @@ private void uploadInOneChunk(
             () -> s3AsyncClient.putObject(
                 putObjectRequestBuilder.build(),
                 AsyncRequestBody.fromInputStream(
-                    inputStreamContainer.getInputStream(),
+                    // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
+                    // data can be retried instead of retrying whole file by the application.
+                    new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1)),
                     inputStreamContainer.getContentLength(),
                     streamReadExecutor
                 )

From 1715da57f5c6223bf590118bf5adad8a7afaf9da Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan 
Date: Wed, 18 Oct 2023 18:23:24 +0530
Subject: [PATCH 179/388] Repository stats for remote store (#10567)

Signed-off-by: Bukhtawar Khan 
---
 CHANGELOG.md                                  |   1 +
 .../s3/S3BlobStoreRepositoryTests.java        |  75 ++++++++
 .../repositories/s3/S3BlobContainer.java      |   5 +-
 .../repositories/s3/S3BlobStore.java          |  12 ++
 .../repositories/s3/StatsMetricPublisher.java | 182 +++++++++++++-----
 .../s3/async/AsyncTransferManager.java        |  24 ++-
 .../s3/async/AsyncTransferManagerTests.java   |  13 +-
 .../admin/cluster/node/stats/NodeStats.java   |  24 ++-
 .../cluster/node/stats/NodesStatsRequest.java |   3 +-
 .../node/stats/TransportNodesStatsAction.java |   3 +-
 .../stats/TransportClusterStatsAction.java    |   1 +
 .../common/blobstore/BlobStore.java           |  31 +++
 .../common/blobstore/EncryptedBlobStore.java  |  10 +
 .../main/java/org/opensearch/node/Node.java   |   3 +-
 .../java/org/opensearch/node/NodeService.java |  12 +-
 .../repositories/RepositoriesService.java     |  19 +-
 .../repositories/RepositoriesStats.java       |  52 +++++
 .../RepositoriesStatsArchive.java             |  14 --
 .../repositories/RepositoryInfo.java          |  54 +-----
 .../repositories/RepositoryStats.java         |  69 ++++++-
 .../repositories/RepositoryStatsSnapshot.java |  22 +--
 .../blobstore/BlobStoreRepository.java        |   2 +
 .../blobstore/MeteredBlobStoreRepository.java |  18 +-
 .../cluster/node/stats/NodeStatsTests.java    |   1 +
 .../opensearch/cluster/DiskUsageTests.java    |   6 +
 .../RepositoriesServiceTests.java             |  13 +-
 .../RepositoriesStatsArchiveTests.java        |   8 +-
 .../MockInternalClusterInfoService.java       |   3 +-
 ...chMockAPIBasedRepositoryIntegTestCase.java |   2 +-
 .../opensearch/test/InternalTestCluster.java  |   1 +
 30 files changed, 479 insertions(+), 204 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/repositories/RepositoriesStats.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 243423223ca52..0742215a2dfb9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x]
 ### Added
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
+- [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index 5f88ad7867513..1361f3165b653 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -37,6 +37,8 @@
 
 import software.amazon.awssdk.core.internal.http.pipeline.stages.ApplyTransactionIdStage;
 
+import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.SuppressForbidden;
@@ -51,10 +53,15 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.repositories.Repository;
+import org.opensearch.repositories.RepositoryMissingException;
+import org.opensearch.repositories.RepositoryStats;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.repositories.blobstore.OpenSearchMockAPIBasedRepositoryIntegTestCase;
 import org.opensearch.repositories.s3.utils.AwsRequestSigner;
 import org.opensearch.snapshots.mockstore.BlobStoreWrapper;
+import org.opensearch.test.BackgroundIndexer;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.threadpool.ThreadPool;
 
@@ -63,12 +70,18 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.stream.StreamSupport;
 
 import fixture.s3.S3HttpHandler;
 
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint")
 // Need to set up a new cluster for each test because cluster settings use randomized authentication settings
@@ -152,6 +165,66 @@ protected Settings nodeSettings(int nodeOrdinal) {
         return builder.build();
     }
 
+    @Override
+    public void testRequestStats() throws Exception {
+        final String repository = createRepository(randomName());
+        final String index = "index-no-merges";
+        createIndex(
+            index,
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+
+        final long nbDocs = randomLongBetween(10_000L, 20_000L);
+        try (BackgroundIndexer indexer = new BackgroundIndexer(index, "_doc", client(), (int) nbDocs)) {
+            waitForDocs(nbDocs, indexer);
+        }
+
+        flushAndRefresh(index);
+        ForceMergeResponse forceMerge = client().admin().indices().prepareForceMerge(index).setFlush(true).setMaxNumSegments(1).get();
+        assertThat(forceMerge.getSuccessfulShards(), equalTo(1));
+        assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
+
+        final String snapshot = "snapshot";
+        assertSuccessfulSnapshot(
+            client().admin().cluster().prepareCreateSnapshot(repository, snapshot).setWaitForCompletion(true).setIndices(index)
+        );
+
+        assertAcked(client().admin().indices().prepareDelete(index));
+
+        assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repository, snapshot).setWaitForCompletion(true));
+        ensureGreen(index);
+        assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
+
+        assertAcked(client().admin().cluster().prepareDeleteSnapshot(repository, snapshot).get());
+
+        final RepositoryStats repositoryStats = StreamSupport.stream(
+            internalCluster().getInstances(RepositoriesService.class).spliterator(),
+            false
+        ).map(repositoriesService -> {
+            try {
+                return repositoriesService.repository(repository);
+            } catch (RepositoryMissingException e) {
+                return null;
+            }
+        }).filter(Objects::nonNull).map(Repository::stats).reduce(RepositoryStats::merge).get();
+
+        Map> extendedStats = repositoryStats.extendedStats;
+        Map aggregatedStats = new HashMap<>();
+        extendedStats.forEach((k, v) -> {
+            if (k == BlobStore.Metric.RETRY_COUNT || k == BlobStore.Metric.REQUEST_SUCCESS || k == BlobStore.Metric.REQUEST_FAILURE) {
+                for (Map.Entry entry : v.entrySet()) {
+                    aggregatedStats.merge(entry.getKey(), entry.getValue(), Math::addExact);
+                }
+            }
+
+        });
+        final Map mockCalls = getMockRequestCounts();
+
+        String assertionErrorMsg = String.format("SDK sent [%s] calls and handler measured [%s] calls", aggregatedStats, mockCalls);
+
+        assertEquals(assertionErrorMsg, mockCalls, aggregatedStats);
+    }
+
     /**
      * S3RepositoryPlugin that allows to disable chunked encoding and to set a low threshold between single upload and multipart upload.
      */
@@ -263,6 +336,8 @@ public void maybeTrack(final String request, Headers requestHeaders) {
                 trackRequest("PutMultipartObject");
             } else if (Regex.simpleMatch("PUT /*/*", request)) {
                 trackRequest("PutObject");
+            } else if (Regex.simpleMatch("POST /*?delete*", request)) {
+                trackRequest("DeleteObjects");
             }
         }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
index 9777bd974d56c..24aee99242957 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
@@ -199,7 +199,7 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener comp
                     ? amazonS3Reference.get().priorityClient()
                     : amazonS3Reference.get().client();
                 CompletableFuture completableFuture = blobStore.getAsyncTransferManager()
-                    .uploadObject(s3AsyncClient, uploadRequest, streamContext);
+                    .uploadObject(s3AsyncClient, uploadRequest, streamContext, blobStore.getStatsMetricPublisher());
                 completableFuture.whenComplete((response, throwable) -> {
                     if (throwable == null) {
                         completionListener.onResponse(response);
@@ -384,7 +384,7 @@ private void doDeleteBlobs(List blobNames, boolean relative) throws IOEx
         assert outstanding.isEmpty();
     }
 
-    private static DeleteObjectsRequest bulkDelete(String bucket, List blobs) {
+    private DeleteObjectsRequest bulkDelete(String bucket, List blobs) {
         return DeleteObjectsRequest.builder()
             .bucket(bucket)
             .delete(
@@ -393,6 +393,7 @@ private static DeleteObjectsRequest bulkDelete(String bucket, List blobs
                     .quiet(true)
                     .build()
             )
+            .overrideConfiguration(o -> o.addMetricPublisher(blobStore.getStatsMetricPublisher().deleteObjectsMetricPublisher))
             .build();
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
index 80005d92344a4..f568d871dd31a 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
@@ -47,6 +47,8 @@
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 
@@ -180,6 +182,16 @@ public Map stats() {
         return statsMetricPublisher.getStats().toMap();
     }
 
+    @Override
+    public Map> extendedStats() {
+        if (statsMetricPublisher.getExtendedStats() == null || statsMetricPublisher.getExtendedStats().isEmpty()) {
+            return Collections.emptyMap();
+        }
+        Map> extendedStats = new HashMap<>();
+        statsMetricPublisher.getExtendedStats().forEach((k, v) -> extendedStats.put(k, v.toMap()));
+        return extendedStats;
+    }
+
     public ObjectCannedACL getCannedACL() {
         return cannedACL;
     }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java
index cad0037f99249..0c63bfdb1ff97 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java
@@ -8,10 +8,13 @@
 
 package org.opensearch.repositories.s3;
 
-import software.amazon.awssdk.http.HttpMetric;
 import software.amazon.awssdk.metrics.MetricCollection;
 import software.amazon.awssdk.metrics.MetricPublisher;
+import software.amazon.awssdk.metrics.MetricRecord;
 
+import org.opensearch.common.blobstore.BlobStore;
+
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
@@ -20,18 +23,67 @@ public class StatsMetricPublisher {
 
     private final Stats stats = new Stats();
 
+    private final Map extendedStats = new HashMap<>() {
+        {
+            put(BlobStore.Metric.REQUEST_LATENCY, new Stats());
+            put(BlobStore.Metric.REQUEST_SUCCESS, new Stats());
+            put(BlobStore.Metric.REQUEST_FAILURE, new Stats());
+            put(BlobStore.Metric.RETRY_COUNT, new Stats());
+        }
+    };
+
     public MetricPublisher listObjectsMetricPublisher = new MetricPublisher() {
         @Override
         public void publish(MetricCollection metricCollection) {
-            stats.listCount.addAndGet(
-                metricCollection.children()
-                    .stream()
-                    .filter(
-                        metricRecords -> metricRecords.name().equals("ApiCallAttempt")
-                            && !metricRecords.metricValues(HttpMetric.HTTP_STATUS_CODE).isEmpty()
-                    )
-                    .count()
-            );
+            for (MetricRecord metricRecord : metricCollection) {
+                switch (metricRecord.metric().name()) {
+                    case "ApiCallDuration":
+                        extendedStats.get(BlobStore.Metric.REQUEST_LATENCY).listMetrics.addAndGet(
+                            ((Duration) metricRecord.value()).toMillis()
+                        );
+                        break;
+                    case "RetryCount":
+                        extendedStats.get(BlobStore.Metric.RETRY_COUNT).listMetrics.addAndGet(((Integer) metricRecord.value()));
+                        break;
+                    case "ApiCallSuccessful":
+                        if ((Boolean) metricRecord.value()) {
+                            extendedStats.get(BlobStore.Metric.REQUEST_SUCCESS).listMetrics.addAndGet(1);
+                        } else {
+                            extendedStats.get(BlobStore.Metric.REQUEST_FAILURE).listMetrics.addAndGet(1);
+                        }
+                        stats.listMetrics.addAndGet(1);
+                        break;
+                }
+            }
+        }
+
+        @Override
+        public void close() {}
+    };
+
+    public MetricPublisher deleteObjectsMetricPublisher = new MetricPublisher() {
+        @Override
+        public void publish(MetricCollection metricCollection) {
+            for (MetricRecord metricRecord : metricCollection) {
+                switch (metricRecord.metric().name()) {
+                    case "ApiCallDuration":
+                        extendedStats.get(BlobStore.Metric.REQUEST_LATENCY).deleteMetrics.addAndGet(
+                            ((Duration) metricRecord.value()).toMillis()
+                        );
+                        break;
+                    case "RetryCount":
+                        extendedStats.get(BlobStore.Metric.RETRY_COUNT).deleteMetrics.addAndGet(((Integer) metricRecord.value()));
+                        break;
+                    case "ApiCallSuccessful":
+                        if ((Boolean) metricRecord.value()) {
+                            extendedStats.get(BlobStore.Metric.REQUEST_SUCCESS).deleteMetrics.addAndGet(1);
+                        } else {
+                            extendedStats.get(BlobStore.Metric.REQUEST_FAILURE).deleteMetrics.addAndGet(1);
+                        }
+                        stats.deleteMetrics.addAndGet(1);
+                        break;
+                }
+            }
         }
 
         @Override
@@ -41,15 +93,26 @@ public void close() {}
     public MetricPublisher getObjectMetricPublisher = new MetricPublisher() {
         @Override
         public void publish(MetricCollection metricCollection) {
-            stats.getCount.addAndGet(
-                metricCollection.children()
-                    .stream()
-                    .filter(
-                        metricRecords -> metricRecords.name().equals("ApiCallAttempt")
-                            && !metricRecords.metricValues(HttpMetric.HTTP_STATUS_CODE).isEmpty()
-                    )
-                    .count()
-            );
+            for (MetricRecord metricRecord : metricCollection) {
+                switch (metricRecord.metric().name()) {
+                    case "ApiCallDuration":
+                        extendedStats.get(BlobStore.Metric.REQUEST_LATENCY).getMetrics.addAndGet(
+                            ((Duration) metricRecord.value()).toMillis()
+                        );
+                        break;
+                    case "RetryCount":
+                        extendedStats.get(BlobStore.Metric.RETRY_COUNT).getMetrics.addAndGet(((Integer) metricRecord.value()));
+                        break;
+                    case "ApiCallSuccessful":
+                        if ((Boolean) metricRecord.value()) {
+                            extendedStats.get(BlobStore.Metric.REQUEST_SUCCESS).getMetrics.addAndGet(1);
+                        } else {
+                            extendedStats.get(BlobStore.Metric.REQUEST_FAILURE).getMetrics.addAndGet(1);
+                        }
+                        stats.getMetrics.addAndGet(1);
+                        break;
+                }
+            }
         }
 
         @Override
@@ -59,15 +122,26 @@ public void close() {}
     public MetricPublisher putObjectMetricPublisher = new MetricPublisher() {
         @Override
         public void publish(MetricCollection metricCollection) {
-            stats.putCount.addAndGet(
-                metricCollection.children()
-                    .stream()
-                    .filter(
-                        metricRecords -> metricRecords.name().equals("ApiCallAttempt")
-                            && !metricRecords.metricValues(HttpMetric.HTTP_STATUS_CODE).isEmpty()
-                    )
-                    .count()
-            );
+            for (MetricRecord metricRecord : metricCollection) {
+                switch (metricRecord.metric().name()) {
+                    case "ApiCallDuration":
+                        extendedStats.get(BlobStore.Metric.REQUEST_LATENCY).putMetrics.addAndGet(
+                            ((Duration) metricRecord.value()).toMillis()
+                        );
+                        break;
+                    case "RetryCount":
+                        extendedStats.get(BlobStore.Metric.RETRY_COUNT).putMetrics.addAndGet(((Integer) metricRecord.value()));
+                        break;
+                    case "ApiCallSuccessful":
+                        if ((Boolean) metricRecord.value()) {
+                            extendedStats.get(BlobStore.Metric.REQUEST_SUCCESS).putMetrics.addAndGet(1);
+                        } else {
+                            extendedStats.get(BlobStore.Metric.REQUEST_FAILURE).putMetrics.addAndGet(1);
+                        }
+                        stats.putMetrics.addAndGet(1);
+                        break;
+                }
+            }
         }
 
         @Override
@@ -77,15 +151,26 @@ public void close() {}
     public MetricPublisher multipartUploadMetricCollector = new MetricPublisher() {
         @Override
         public void publish(MetricCollection metricCollection) {
-            stats.postCount.addAndGet(
-                metricCollection.children()
-                    .stream()
-                    .filter(
-                        metricRecords -> metricRecords.name().equals("ApiCallAttempt")
-                            && !metricRecords.metricValues(HttpMetric.HTTP_STATUS_CODE).isEmpty()
-                    )
-                    .count()
-            );
+            for (MetricRecord metricRecord : metricCollection) {
+                switch (metricRecord.metric().name()) {
+                    case "ApiCallDuration":
+                        extendedStats.get(BlobStore.Metric.REQUEST_LATENCY).multiPartPutMetrics.addAndGet(
+                            ((Duration) metricRecord.value()).toMillis()
+                        );
+                        break;
+                    case "RetryCount":
+                        extendedStats.get(BlobStore.Metric.RETRY_COUNT).multiPartPutMetrics.addAndGet(((Integer) metricRecord.value()));
+                        break;
+                    case "ApiCallSuccessful":
+                        if ((Boolean) metricRecord.value()) {
+                            extendedStats.get(BlobStore.Metric.REQUEST_SUCCESS).multiPartPutMetrics.addAndGet(1);
+                        } else {
+                            extendedStats.get(BlobStore.Metric.REQUEST_FAILURE).multiPartPutMetrics.addAndGet(1);
+                        }
+                        stats.multiPartPutMetrics.addAndGet(1);
+                        break;
+                }
+            }
         }
 
         @Override
@@ -96,22 +181,29 @@ public Stats getStats() {
         return stats;
     }
 
+    public Map getExtendedStats() {
+        return extendedStats;
+    }
+
     static class Stats {
 
-        final AtomicLong listCount = new AtomicLong();
+        final AtomicLong listMetrics = new AtomicLong();
+
+        final AtomicLong getMetrics = new AtomicLong();
 
-        final AtomicLong getCount = new AtomicLong();
+        final AtomicLong putMetrics = new AtomicLong();
 
-        final AtomicLong putCount = new AtomicLong();
+        final AtomicLong deleteMetrics = new AtomicLong();
 
-        final AtomicLong postCount = new AtomicLong();
+        final AtomicLong multiPartPutMetrics = new AtomicLong();
 
         Map toMap() {
             final Map results = new HashMap<>();
-            results.put("GetObject", getCount.get());
-            results.put("ListObjects", listCount.get());
-            results.put("PutObject", putCount.get());
-            results.put("PutMultipartObject", postCount.get());
+            results.put("GetObject", getMetrics.get());
+            results.put("ListObjects", listMetrics.get());
+            results.put("PutObject", putMetrics.get());
+            results.put("DeleteObjects", deleteMetrics.get());
+            results.put("PutMultipartObject", multiPartPutMetrics.get());
             return results;
         }
     }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
index fb1e409388a69..db04636b89d50 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
@@ -35,6 +35,7 @@
 import org.opensearch.common.util.ByteUtils;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.s3.SocketAccess;
+import org.opensearch.repositories.s3.StatsMetricPublisher;
 import org.opensearch.repositories.s3.io.CheckedContainer;
 
 import java.io.BufferedInputStream;
@@ -87,16 +88,21 @@ public AsyncTransferManager(long minimumPartSize, ExecutorService executorServic
      * @param streamContext The {@link StreamContext} to supply streams during upload
      * @return A {@link CompletableFuture} to listen for upload completion
      */
-    public CompletableFuture uploadObject(S3AsyncClient s3AsyncClient, UploadRequest uploadRequest, StreamContext streamContext) {
+    public CompletableFuture uploadObject(
+        S3AsyncClient s3AsyncClient,
+        UploadRequest uploadRequest,
+        StreamContext streamContext,
+        StatsMetricPublisher statsMetricPublisher
+    ) {
 
         CompletableFuture returnFuture = new CompletableFuture<>();
         try {
             if (streamContext.getNumberOfParts() == 1) {
                 log.debug(() -> "Starting the upload as a single upload part request");
-                uploadInOneChunk(s3AsyncClient, uploadRequest, streamContext.provideStream(0), returnFuture);
+                uploadInOneChunk(s3AsyncClient, uploadRequest, streamContext.provideStream(0), returnFuture, statsMetricPublisher);
             } else {
                 log.debug(() -> "Starting the upload as multipart upload request");
-                uploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture);
+                uploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture, statsMetricPublisher);
             }
         } catch (Throwable throwable) {
             returnFuture.completeExceptionally(throwable);
@@ -109,12 +115,14 @@ private void uploadInParts(
         S3AsyncClient s3AsyncClient,
         UploadRequest uploadRequest,
         StreamContext streamContext,
-        CompletableFuture returnFuture
+        CompletableFuture returnFuture,
+        StatsMetricPublisher statsMetricPublisher
     ) {
 
         CreateMultipartUploadRequest.Builder createMultipartUploadRequestBuilder = CreateMultipartUploadRequest.builder()
             .bucket(uploadRequest.getBucket())
-            .key(uploadRequest.getKey());
+            .key(uploadRequest.getKey())
+            .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.multipartUploadMetricCollector));
         if (uploadRequest.doRemoteDataIntegrityCheck()) {
             createMultipartUploadRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
         }
@@ -287,12 +295,14 @@ private void uploadInOneChunk(
         S3AsyncClient s3AsyncClient,
         UploadRequest uploadRequest,
         InputStreamContainer inputStreamContainer,
-        CompletableFuture returnFuture
+        CompletableFuture returnFuture,
+        StatsMetricPublisher statsMetricPublisher
     ) {
         PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
             .bucket(uploadRequest.getBucket())
             .key(uploadRequest.getKey())
-            .contentLength(uploadRequest.getContentLength());
+            .contentLength(uploadRequest.getContentLength())
+            .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.putObjectMetricPublisher));
         if (uploadRequest.doRemoteDataIntegrityCheck()) {
             putObjectRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
             putObjectRequestBuilder.checksumCRC32(base64StringFromLong(uploadRequest.getExpectedChecksum()));
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
index 9c07b929052bc..607453986ab16 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
@@ -33,6 +33,7 @@
 import org.opensearch.common.io.InputStreamContainer;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.blobstore.ZeroInputStream;
+import org.opensearch.repositories.s3.StatsMetricPublisher;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
@@ -80,7 +81,8 @@ public void testOneChunkUpload() {
                 ByteSizeUnit.MB.toBytes(1),
                 ByteSizeUnit.MB.toBytes(1),
                 1
-            )
+            ),
+            new StatsMetricPublisher()
         );
 
         try {
@@ -118,7 +120,8 @@ public void testOneChunkUploadCorruption() {
                 ByteSizeUnit.MB.toBytes(1),
                 ByteSizeUnit.MB.toBytes(1),
                 1
-            )
+            ),
+            new StatsMetricPublisher()
         );
 
         try {
@@ -169,7 +172,8 @@ public void testMultipartUpload() {
                 ByteSizeUnit.MB.toBytes(1),
                 ByteSizeUnit.MB.toBytes(1),
                 5
-            )
+            ),
+            new StatsMetricPublisher()
         );
 
         try {
@@ -219,7 +223,8 @@ public void testMultipartUploadCorruption() {
                 ByteSizeUnit.MB.toBytes(1),
                 ByteSizeUnit.MB.toBytes(1),
                 5
-            )
+            ),
+            new StatsMetricPublisher()
         );
 
         try {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index 69efea186d927..6ce6ca40cbce4 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -57,6 +57,7 @@
 import org.opensearch.monitor.process.ProcessStats;
 import org.opensearch.node.AdaptiveSelectionStats;
 import org.opensearch.node.NodesResourceUsageStats;
+import org.opensearch.repositories.RepositoriesStats;
 import org.opensearch.script.ScriptCacheStats;
 import org.opensearch.script.ScriptStats;
 import org.opensearch.search.backpressure.stats.SearchBackpressureStats;
@@ -146,6 +147,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private NodesResourceUsageStats resourceUsageStats;
 
+    @Nullable
+    private RepositoriesStats repositoriesStats;
+
     public NodeStats(StreamInput in) throws IOException {
         super(in);
         timestamp = in.readVLong();
@@ -207,6 +211,11 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             resourceUsageStats = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            repositoriesStats = in.readOptionalWriteable(RepositoriesStats::new);
+        } else {
+            repositoriesStats = null;
+        }
     }
 
     public NodeStats(
@@ -234,7 +243,8 @@ public NodeStats(
         @Nullable WeightedRoutingStats weightedRoutingStats,
         @Nullable FileCacheStats fileCacheStats,
         @Nullable TaskCancellationStats taskCancellationStats,
-        @Nullable SearchPipelineStats searchPipelineStats
+        @Nullable SearchPipelineStats searchPipelineStats,
+        @Nullable RepositoriesStats repositoriesStats
     ) {
         super(node);
         this.timestamp = timestamp;
@@ -261,6 +271,7 @@ public NodeStats(
         this.fileCacheStats = fileCacheStats;
         this.taskCancellationStats = taskCancellationStats;
         this.searchPipelineStats = searchPipelineStats;
+        this.repositoriesStats = repositoriesStats;
     }
 
     public long getTimestamp() {
@@ -403,6 +414,11 @@ public SearchPipelineStats getSearchPipelineStats() {
         return searchPipelineStats;
     }
 
+    @Nullable
+    public RepositoriesStats getRepositoriesStats() {
+        return repositoriesStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -449,6 +465,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_3_0_0)) { // make it 2.12 when we backport
             out.writeOptionalWriteable(resourceUsageStats);
         }
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalWriteable(repositoriesStats);
+        }
     }
 
     @Override
@@ -542,6 +561,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getResourceUsageStats() != null) {
             getResourceUsageStats().toXContent(builder, params);
         }
+        if (getRepositoriesStats() != null) {
+            getRepositoriesStats().toXContent(builder, params);
+        }
         return builder;
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index 99c9fb2d1e26a..88dff20354aa2 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -214,7 +214,8 @@ public enum Metric {
         FILE_CACHE_STATS("file_cache"),
         TASK_CANCELLATION("task_cancellation"),
         SEARCH_PIPELINE("search_pipeline"),
-        RESOURCE_USAGE_STATS("resource_usage_stats");
+        RESOURCE_USAGE_STATS("resource_usage_stats"),
+        REPOSITORIES("repositories");
 
         private String metricName;
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 204157236a282..aa02f8e580f4a 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -125,7 +125,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics),
             NodesStatsRequest.Metric.TASK_CANCELLATION.containedIn(metrics),
             NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics),
-            NodesStatsRequest.Metric.RESOURCE_USAGE_STATS.containedIn(metrics)
+            NodesStatsRequest.Metric.RESOURCE_USAGE_STATS.containedIn(metrics),
+            NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics)
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index d8323e209be23..f51fabbfb2388 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -169,6 +169,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List shardsStats = new ArrayList<>();
diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
index 2ee3e9557b354..0f6646d37f950 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
@@ -49,6 +49,9 @@ public interface BlobStore extends Closeable {
      */
     BlobContainer blobContainer(BlobPath path);
 
+    /**
+     * Returns statistics on the count of operations that have been performed on this blob store
+     */
     /**
      * Returns statistics on the count of operations that have been performed on this blob store
      */
@@ -56,8 +59,36 @@ default Map stats() {
         return Collections.emptyMap();
     }
 
+    /**
+     * Returns details statistics of operations that have been performed on this blob store
+     */
+    default Map> extendedStats() {
+        return Collections.emptyMap();
+    }
+
     /**
      * Reload the blob store inplace
      */
     default void reload(RepositoryMetadata repositoryMetadata) {}
+
+    /**
+     * Metrics for BlobStore interactions
+     */
+    enum Metric {
+        REQUEST_SUCCESS("request_success_total"),
+        REQUEST_FAILURE("request_failures_total"),
+        REQUEST_LATENCY("request_time_in_millis"),
+        RETRY_COUNT("request_retry_count_total");
+
+        private String metricName;
+
+        Metric(String name) {
+            this.metricName = name;
+        }
+
+        public String metricName() {
+            return this.metricName;
+        }
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java
index 4d2d69e473438..a18ca8b9d5c39 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/EncryptedBlobStore.java
@@ -75,6 +75,16 @@ public Map stats() {
         return blobStore.stats();
     }
 
+    /**
+     * Retrieves extended statistics about the BlobStore. Delegates the call to the underlying BlobStore's extendedStats() method.
+     *
+     * @return A map containing extended statistics about the BlobStore.
+     */
+    @Override
+    public Map> extendedStats() {
+        return blobStore.extendedStats();
+    }
+
     /**
      * Closes the EncryptedBlobStore by decrementing the reference count of the CryptoManager and closing the
      * underlying BlobStore. This ensures proper cleanup of resources.
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 69b80462bbf0b..c9148f382a028 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -1115,7 +1115,8 @@ protected Node(
                 searchPipelineService,
                 fileCache,
                 taskCancellationMonitoringService,
-                resourceUsageCollectorService
+                resourceUsageCollectorService,
+                repositoryService
             );
 
             final SearchService searchService = newSearchService(
diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java
index 9bb07080fa717..e2d7bc2c86ba3 100644
--- a/server/src/main/java/org/opensearch/node/NodeService.java
+++ b/server/src/main/java/org/opensearch/node/NodeService.java
@@ -53,6 +53,7 @@
 import org.opensearch.ingest.IngestService;
 import org.opensearch.monitor.MonitorService;
 import org.opensearch.plugins.PluginsService;
+import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.script.ScriptService;
 import org.opensearch.search.aggregations.support.AggregationUsageService;
 import org.opensearch.search.backpressure.SearchBackpressureService;
@@ -93,6 +94,7 @@ public class NodeService implements Closeable {
     private final Discovery discovery;
     private final FileCache fileCache;
     private final TaskCancellationMonitoringService taskCancellationMonitoringService;
+    private final RepositoriesService repositoriesService;
 
     NodeService(
         Settings settings,
@@ -116,7 +118,8 @@ public class NodeService implements Closeable {
         SearchPipelineService searchPipelineService,
         FileCache fileCache,
         TaskCancellationMonitoringService taskCancellationMonitoringService,
-        ResourceUsageCollectorService resourceUsageCollectorService
+        ResourceUsageCollectorService resourceUsageCollectorService,
+        RepositoriesService repositoriesService
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -140,6 +143,7 @@ public class NodeService implements Closeable {
         this.fileCache = fileCache;
         this.taskCancellationMonitoringService = taskCancellationMonitoringService;
         this.resourceUsageCollectorService = resourceUsageCollectorService;
+        this.repositoriesService = repositoriesService;
         clusterService.addStateApplier(ingestService);
         clusterService.addStateApplier(searchPipelineService);
     }
@@ -221,7 +225,8 @@ public NodeStats stats(
         boolean fileCacheStats,
         boolean taskCancellation,
         boolean searchPipelineStats,
-        boolean resourceUsageStats
+        boolean resourceUsageStats,
+        boolean repositoriesStats
     ) {
         // for indices stats we want to include previous allocated shards stats as well (it will
         // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
@@ -250,7 +255,8 @@ public NodeStats stats(
             weightedRoutingStats ? WeightedRoutingStats.getInstance() : null,
             fileCacheStats && fileCache != null ? fileCache.fileCacheStats() : null,
             taskCancellation ? this.taskCancellationMonitoringService.stats() : null,
-            searchPipelineStats ? this.searchPipelineService.stats() : null
+            searchPipelineStats ? this.searchPipelineService.stats() : null,
+            repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java
index 72266c053a1ae..68669feb16abc 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java
@@ -457,7 +457,6 @@ public void applyClusterState(ClusterChangedEvent event) {
                     logger.debug("unregistering repository [{}]", entry.getKey());
                     Repository repository = entry.getValue();
                     closeRepository(repository);
-                    archiveRepositoryStats(repository, state.version());
                 } else {
                     survivors.put(entry.getKey(), entry.getValue());
                 }
@@ -485,7 +484,6 @@ public void applyClusterState(ClusterChangedEvent event) {
                             } else {
                                 logger.debug("updating repository [{}]", repositoryMetadata.name());
                                 closeRepository(repository);
-                                archiveRepositoryStats(repository, state.version());
                                 repository = null;
                                 try {
                                     repository = createRepository(repositoryMetadata, typesRegistry);
@@ -575,12 +573,12 @@ public Repository repository(String repositoryName) {
     }
 
     public List repositoriesStats() {
-        List archivedRepoStats = repositoriesStatsArchive.getArchivedStats();
         List activeRepoStats = getRepositoryStatsForActiveRepositories();
+        return activeRepoStats;
+    }
 
-        List repositoriesStats = new ArrayList<>(archivedRepoStats);
-        repositoriesStats.addAll(activeRepoStats);
-        return repositoriesStats;
+    public RepositoriesStats getRepositoriesStats() {
+        return new RepositoriesStats(repositoriesStats());
     }
 
     private List getRepositoryStatsForActiveRepositories() {
@@ -640,15 +638,6 @@ public void closeRepository(Repository repository) {
         repository.close();
     }
 
-    private void archiveRepositoryStats(Repository repository, long clusterStateVersion) {
-        if (repository instanceof MeteredBlobStoreRepository) {
-            RepositoryStatsSnapshot stats = ((MeteredBlobStoreRepository) repository).statsSnapshotForArchival(clusterStateVersion);
-            if (repositoriesStatsArchive.archive(stats) == false) {
-                logger.warn("Unable to archive the repository stats [{}] as the archive is full.", stats);
-            }
-        }
-    }
-
     /**
      * Creates repository holder. This method starts the non-internal repository
      */
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesStats.java b/server/src/main/java/org/opensearch/repositories/RepositoriesStats.java
new file mode 100644
index 0000000000000..b24e0dddd852a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/repositories/RepositoriesStats.java
@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.util.CollectionUtils;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Encapsulates stats for multiple repositories*
+ */
+public class RepositoriesStats implements Writeable, ToXContentObject {
+
+    List repositoryStatsSnapshots;
+
+    public RepositoriesStats(List repositoryStatsSnapshots) {
+        this.repositoryStatsSnapshots = repositoryStatsSnapshots;
+    }
+
+    public RepositoriesStats(StreamInput in) throws IOException {
+        this.repositoryStatsSnapshots = in.readList(RepositoryStatsSnapshot::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeList(repositoryStatsSnapshots);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray("repositories");
+        if (CollectionUtils.isEmpty(repositoryStatsSnapshots) == false) {
+            for (RepositoryStatsSnapshot repositoryStatsSnapshot : repositoryStatsSnapshots) {
+                repositoryStatsSnapshot.toXContent(builder, params);
+            }
+        }
+        builder.endArray();
+        return builder;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesStatsArchive.java b/server/src/main/java/org/opensearch/repositories/RepositoriesStatsArchive.java
index b8f100706f81e..3d35f75176eaf 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoriesStatsArchive.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoriesStatsArchive.java
@@ -70,11 +70,6 @@ public RepositoriesStatsArchive(TimeValue retentionPeriod, int maxCapacity, Long
      * @return {@code true} if the repository stats were archived, {@code false} otherwise.
      */
     synchronized boolean archive(final RepositoryStatsSnapshot repositoryStats) {
-        assert containsRepositoryStats(repositoryStats) == false : "A repository with ephemeral id "
-            + repositoryStats.getRepositoryInfo().ephemeralId
-            + " is already archived";
-        assert repositoryStats.isArchived();
-
         evict();
 
         if (archive.size() >= maxCapacity) {
@@ -116,15 +111,6 @@ private void evict() {
         }
     }
 
-    private boolean containsRepositoryStats(RepositoryStatsSnapshot repositoryStats) {
-        return archive.stream()
-            .anyMatch(
-                entry -> entry.repositoryStatsSnapshot.getRepositoryInfo().ephemeralId.equals(
-                    repositoryStats.getRepositoryInfo().ephemeralId
-                )
-            );
-    }
-
     private static class ArchiveEntry {
         private final RepositoryStatsSnapshot repositoryStatsSnapshot;
         private final long createdAtMillis;
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java b/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java
index 8aa86fc46d591..387a685bd6526 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoryInfo.java
@@ -32,7 +32,6 @@
 
 package org.opensearch.repositories;
 
-import org.opensearch.common.Nullable;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -51,64 +50,27 @@
  * @opensearch.internal
  */
 public final class RepositoryInfo implements Writeable, ToXContentFragment {
-    public final String ephemeralId;
     public final String name;
     public final String type;
     public final Map location;
-    public final long startedAt;
-    @Nullable
-    public final Long stoppedAt;
 
-    public RepositoryInfo(String ephemeralId, String name, String type, Map location, long startedAt) {
-        this(ephemeralId, name, type, location, startedAt, null);
-    }
-
-    public RepositoryInfo(
-        String ephemeralId,
-        String name,
-        String type,
-        Map location,
-        long startedAt,
-        @Nullable Long stoppedAt
-    ) {
-        this.ephemeralId = ephemeralId;
+    public RepositoryInfo(String name, String type, Map location) {
         this.name = name;
         this.type = type;
         this.location = location;
-        this.startedAt = startedAt;
-        if (stoppedAt != null && startedAt > stoppedAt) {
-            throw new IllegalArgumentException("createdAt must be before or equal to stoppedAt");
-        }
-        this.stoppedAt = stoppedAt;
     }
 
     public RepositoryInfo(StreamInput in) throws IOException {
-        this.ephemeralId = in.readString();
         this.name = in.readString();
         this.type = in.readString();
         this.location = in.readMap(StreamInput::readString, StreamInput::readString);
-        this.startedAt = in.readLong();
-        this.stoppedAt = in.readOptionalLong();
-    }
-
-    public RepositoryInfo stopped(long stoppedAt) {
-        assert isStopped() == false : "The repository is already stopped";
-
-        return new RepositoryInfo(ephemeralId, name, type, location, startedAt, stoppedAt);
-    }
-
-    public boolean isStopped() {
-        return stoppedAt != null;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(ephemeralId);
         out.writeString(name);
         out.writeString(type);
         out.writeMap(location, StreamOutput::writeString, StreamOutput::writeString);
-        out.writeLong(startedAt);
-        out.writeOptionalLong(stoppedAt);
     }
 
     @Override
@@ -116,11 +78,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("repository_name", name);
         builder.field("repository_type", type);
         builder.field("repository_location", location);
-        builder.field("repository_ephemeral_id", ephemeralId);
-        builder.field("repository_started_at", startedAt);
-        if (stoppedAt != null) {
-            builder.field("repository_stopped_at", stoppedAt);
-        }
         return builder;
     }
 
@@ -129,17 +86,12 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         RepositoryInfo that = (RepositoryInfo) o;
-        return ephemeralId.equals(that.ephemeralId)
-            && name.equals(that.name)
-            && type.equals(that.type)
-            && location.equals(that.location)
-            && startedAt == that.startedAt
-            && Objects.equals(stoppedAt, that.stoppedAt);
+        return name.equals(that.name) && type.equals(that.type) && location.equals(that.location);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(ephemeralId, name, type, location, startedAt, stoppedAt);
+        return Objects.hash(name, type, location);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryStats.java b/server/src/main/java/org/opensearch/repositories/RepositoryStats.java
index efd5d6f8560b6..ab97c5eaa1f7a 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoryStats.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoryStats.java
@@ -32,9 +32,13 @@
 
 package org.opensearch.repositories;
 
+import org.opensearch.common.Nullable;
+import org.opensearch.common.blobstore.BlobStore;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -47,32 +51,63 @@
  *
  * @opensearch.internal
  */
-public class RepositoryStats implements Writeable {
+public class RepositoryStats implements Writeable, ToXContentFragment {
 
     public static final RepositoryStats EMPTY_STATS = new RepositoryStats(Collections.emptyMap());
 
+    @Nullable
     public final Map requestCounts;
+    @Nullable
+    public final Map> extendedStats;
+    public final boolean detailed;
 
     public RepositoryStats(Map requestCounts) {
         this.requestCounts = Collections.unmodifiableMap(requestCounts);
+        this.extendedStats = Collections.emptyMap();
+        this.detailed = false;
+    }
+
+    public RepositoryStats(Map> extendedStats, boolean detailed) {
+        this.requestCounts = Collections.emptyMap();
+        this.extendedStats = Collections.unmodifiableMap(extendedStats);
+        this.detailed = detailed;
     }
 
     public RepositoryStats(StreamInput in) throws IOException {
         this.requestCounts = in.readMap(StreamInput::readString, StreamInput::readLong);
+        this.extendedStats = in.readMap(
+            e -> e.readEnum(BlobStore.Metric.class),
+            i -> i.readMap(StreamInput::readString, StreamInput::readLong)
+        );
+        this.detailed = in.readBoolean();
     }
 
     public RepositoryStats merge(RepositoryStats otherStats) {
-        final Map result = new HashMap<>();
-        result.putAll(requestCounts);
-        for (Map.Entry entry : otherStats.requestCounts.entrySet()) {
-            result.merge(entry.getKey(), entry.getValue(), Math::addExact);
+        assert this.detailed == otherStats.detailed;
+        if (detailed) {
+            final Map> result = new HashMap<>();
+            result.putAll(extendedStats);
+            for (Map.Entry> entry : otherStats.extendedStats.entrySet()) {
+                for (Map.Entry nested : entry.getValue().entrySet()) {
+                    result.get(entry.getKey()).merge(nested.getKey(), nested.getValue(), Math::addExact);
+                }
+            }
+            return new RepositoryStats(result, true);
+        } else {
+            final Map result = new HashMap<>();
+            result.putAll(requestCounts);
+            for (Map.Entry entry : otherStats.requestCounts.entrySet()) {
+                result.merge(entry.getKey(), entry.getValue(), Math::addExact);
+            }
+            return new RepositoryStats(result);
         }
-        return new RepositoryStats(result);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(requestCounts, StreamOutput::writeString, StreamOutput::writeLong);
+        out.writeMap(extendedStats, StreamOutput::writeEnum, (o, v) -> o.writeMap(v, StreamOutput::writeString, StreamOutput::writeLong));
+        out.writeBoolean(detailed);
     }
 
     @Override
@@ -80,16 +115,32 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         RepositoryStats that = (RepositoryStats) o;
-        return requestCounts.equals(that.requestCounts);
+        return requestCounts.equals(that.requestCounts) && extendedStats.equals(that.extendedStats) && detailed == that.detailed;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(requestCounts);
+        return Objects.hash(requestCounts, detailed, extendedStats);
     }
 
     @Override
     public String toString() {
-        return "RepositoryStats{" + "requestCounts=" + requestCounts + '}';
+        return "RepositoryStats{" + "requestCounts=" + requestCounts + "extendedStats=" + extendedStats + "detailed =" + detailed + "}";
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (detailed == false) {
+            builder.field("request_counts", requestCounts);
+        } else {
+            extendedStats.forEach((k, v) -> {
+                try {
+                    builder.field(k.metricName(), v);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        }
+        return builder;
     }
 }
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java b/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java
index 2b061cd2c2cc9..0a727980fad0d 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoryStatsSnapshot.java
@@ -53,21 +53,17 @@ public final class RepositoryStatsSnapshot implements Writeable, ToXContentObjec
     private final RepositoryInfo repositoryInfo;
     private final RepositoryStats repositoryStats;
     private final long clusterVersion;
-    private final boolean archived;
 
-    public RepositoryStatsSnapshot(RepositoryInfo repositoryInfo, RepositoryStats repositoryStats, long clusterVersion, boolean archived) {
-        assert archived != (clusterVersion == UNKNOWN_CLUSTER_VERSION);
+    public RepositoryStatsSnapshot(RepositoryInfo repositoryInfo, RepositoryStats repositoryStats, long clusterVersion) {
         this.repositoryInfo = repositoryInfo;
         this.repositoryStats = repositoryStats;
         this.clusterVersion = clusterVersion;
-        this.archived = archived;
     }
 
     public RepositoryStatsSnapshot(StreamInput in) throws IOException {
         this.repositoryInfo = new RepositoryInfo(in);
         this.repositoryStats = new RepositoryStats(in);
         this.clusterVersion = in.readLong();
-        this.archived = in.readBoolean();
     }
 
     public RepositoryInfo getRepositoryInfo() {
@@ -78,10 +74,6 @@ public RepositoryStats getRepositoryStats() {
         return repositoryStats;
     }
 
-    public boolean isArchived() {
-        return archived;
-    }
-
     public long getClusterVersion() {
         return clusterVersion;
     }
@@ -91,18 +83,13 @@ public void writeTo(StreamOutput out) throws IOException {
         repositoryInfo.writeTo(out);
         repositoryStats.writeTo(out);
         out.writeLong(clusterVersion);
-        out.writeBoolean(archived);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         repositoryInfo.toXContent(builder, params);
-        builder.field("request_counts", repositoryStats.requestCounts);
-        builder.field("archived", archived);
-        if (archived) {
-            builder.field("cluster_version", clusterVersion);
-        }
+        repositoryStats.toXContent(builder, params);
         builder.endObject();
         return builder;
     }
@@ -114,13 +101,12 @@ public boolean equals(Object o) {
         RepositoryStatsSnapshot that = (RepositoryStatsSnapshot) o;
         return repositoryInfo.equals(that.repositoryInfo)
             && repositoryStats.equals(that.repositoryStats)
-            && clusterVersion == that.clusterVersion
-            && archived == that.archived;
+            && clusterVersion == that.clusterVersion;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(repositoryInfo, repositoryStats, clusterVersion, archived);
+        return Objects.hash(repositoryInfo, repositoryStats, clusterVersion);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index c48efa9d3d2ee..8a2260e1f6d90 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -851,6 +851,8 @@ public RepositoryStats stats() {
         final BlobStore store = blobStore.get();
         if (store == null) {
             return RepositoryStats.EMPTY_STATS;
+        } else if (store.extendedStats() != null && store.extendedStats().isEmpty() == false) {
+            return new RepositoryStats(store.extendedStats(), true);
         }
         return new RepositoryStats(store.stats());
     }
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java
index d4921f4e6d2e7..0651ff586d412 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/MeteredBlobStoreRepository.java
@@ -34,12 +34,10 @@
 
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.UUIDs;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.repositories.RepositoryInfo;
 import org.opensearch.repositories.RepositoryStatsSnapshot;
-import org.opensearch.threadpool.ThreadPool;
 
 import java.util.Map;
 
@@ -59,14 +57,7 @@ public MeteredBlobStoreRepository(
         Map location
     ) {
         super(metadata, namedXContentRegistry, clusterService, recoverySettings);
-        ThreadPool threadPool = clusterService.getClusterApplierService().threadPool();
-        this.repositoryInfo = new RepositoryInfo(
-            UUIDs.randomBase64UUID(),
-            metadata.name(),
-            metadata.type(),
-            location,
-            threadPool.absoluteTimeInMillis()
-        );
+        this.repositoryInfo = new RepositoryInfo(metadata.name(), metadata.type(), location);
     }
 
     @Override
@@ -78,11 +69,6 @@ public void reload(RepositoryMetadata repositoryMetadata) {
     }
 
     public RepositoryStatsSnapshot statsSnapshot() {
-        return new RepositoryStatsSnapshot(repositoryInfo, stats(), RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION, false);
-    }
-
-    public RepositoryStatsSnapshot statsSnapshotForArchival(long clusterVersion) {
-        RepositoryInfo stoppedRepoInfo = repositoryInfo.stopped(threadPool.absoluteTimeInMillis());
-        return new RepositoryStatsSnapshot(stoppedRepoInfo, stats(), clusterVersion, true);
+        return new RepositoryStatsSnapshot(repositoryInfo, stats(), RepositoryStatsSnapshot.UNKNOWN_CLUSTER_VERSION);
     }
 }
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 4b865383ee007..3491f18da9550 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -852,6 +852,7 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
             weightedRoutingStats,
             null,
             null,
+            null,
             null
         );
     }
diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
index 64949cf861f70..6f03e87bf5824 100644
--- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
@@ -191,6 +191,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -218,6 +219,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -245,6 +247,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
@@ -303,6 +306,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -330,6 +334,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -357,6 +362,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java
index c4599a6e7a00e..43ebb86fd5342 100644
--- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java
+++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java
@@ -219,20 +219,17 @@ public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() {
         assertThat(repositoriesService.repositoriesStats().size(), equalTo(1));
 
         repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", emptyState(), clusterStateWithRepoTypeA));
-        assertThat(repositoriesService.repositoriesStats().size(), equalTo(1));
+        assertThat(repositoriesService.repositoriesStats().size(), equalTo(0));
 
         ClusterState clusterStateWithRepoTypeB = createClusterStateWithRepo(repoName, MeteredRepositoryTypeB.TYPE);
         repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeB, emptyState()));
 
         List repositoriesStats = repositoriesService.repositoriesStats();
-        assertThat(repositoriesStats.size(), equalTo(2));
+        assertThat(repositoriesStats.size(), equalTo(1));
         RepositoryStatsSnapshot repositoryStatsTypeA = repositoriesStats.get(0);
-        assertThat(repositoryStatsTypeA.getRepositoryInfo().type, equalTo(MeteredRepositoryTypeA.TYPE));
-        assertThat(repositoryStatsTypeA.getRepositoryStats(), equalTo(MeteredRepositoryTypeA.STATS));
+        assertThat(repositoryStatsTypeA.getRepositoryInfo().type, equalTo(MeteredRepositoryTypeB.TYPE));
+        assertThat(repositoryStatsTypeA.getRepositoryStats(), equalTo(MeteredRepositoryTypeB.STATS));
 
-        RepositoryStatsSnapshot repositoryStatsTypeB = repositoriesStats.get(1);
-        assertThat(repositoryStatsTypeB.getRepositoryInfo().type, equalTo(MeteredRepositoryTypeB.TYPE));
-        assertThat(repositoryStatsTypeB.getRepositoryStats(), equalTo(MeteredRepositoryTypeB.STATS));
     }
 
     public void testWithSameKeyProviderNames() {
@@ -258,7 +255,7 @@ public void testWithSameKeyProviderNames() {
             kpTypeA
         );
         repositoriesService.applyClusterState(new ClusterChangedEvent("new repo", clusterStateWithRepoTypeB, emptyState()));
-        assertThat(repositoriesService.repositoriesStats().size(), equalTo(2));
+        assertThat(repositoriesService.repositoriesStats().size(), equalTo(1));
         MeteredRepositoryTypeB repositoryB = (MeteredRepositoryTypeB) repositoriesService.repository("repoName");
         assertNotNull(repositoryB);
         assertNotNull(repository.cryptoHandler);
diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesStatsArchiveTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesStatsArchiveTests.java
index cf0b06a3f7d16..da0cbcb1d4b17 100644
--- a/server/src/test/java/org/opensearch/repositories/RepositoriesStatsArchiveTests.java
+++ b/server/src/test/java/org/opensearch/repositories/RepositoriesStatsArchiveTests.java
@@ -32,7 +32,6 @@
 
 package org.opensearch.repositories;
 
-import org.opensearch.common.UUIDs;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.test.OpenSearchTestCase;
 
@@ -122,14 +121,11 @@ private RepositoryStatsSnapshot createRepositoryStats(RepositoryStats repository
 
     private RepositoryStatsSnapshot createRepositoryStats(RepositoryStats repositoryStats, long clusterVersion) {
         RepositoryInfo repositoryInfo = new RepositoryInfo(
-            UUIDs.randomBase64UUID(),
             randomAlphaOfLength(10),
             randomAlphaOfLength(10),
-            Map.of("bucket", randomAlphaOfLength(10)),
-            System.currentTimeMillis(),
-            null
+            Map.of("bucket", randomAlphaOfLength(10))
         );
-        return new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, clusterVersion, true);
+        return new RepositoryStatsSnapshot(repositoryInfo, repositoryStats, clusterVersion);
     }
 
 }
diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
index a520b6278ea47..60a54110fd0b4 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
@@ -121,7 +121,8 @@ List adjustNodesStats(List nodesStats) {
                 nodeStats.getWeightedRoutingStats(),
                 nodeStats.getFileCacheStats(),
                 nodeStats.getTaskCancellationStats(),
-                nodeStats.getSearchPipelineStats()
+                nodeStats.getSearchPipelineStats(),
+                nodeStats.getRepositoriesStats()
             );
         }).collect(Collectors.toList());
     }
diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java
index dff9b997d87db..faa9d52b105b2 100644
--- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/OpenSearchMockAPIBasedRepositoryIntegTestCase.java
@@ -238,7 +238,7 @@ public void testRequestStats() throws Exception {
         assertEquals(assertionErrorMsg, mockCalls, sdkRequestCounts);
     }
 
-    private Map getMockRequestCounts() {
+    protected Map getMockRequestCounts() {
         for (HttpHandler h : handlers.values()) {
             while (h instanceof DelegatingHttpHandler) {
                 if (h instanceof HttpStatsCollectorHandler) {
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index a290f3a1751a0..898e125b94954 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -2721,6 +2721,7 @@ public void ensureEstimatedStats() {
                     false,
                     false,
                     false,
+                    false,
                     false
                 );
                 assertThat(

From d2c2e20f50c2020fc7906ee5db466b1158ace832 Mon Sep 17 00:00:00 2001
From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>
Date: Wed, 18 Oct 2023 09:39:08 -0400
Subject: [PATCH 180/388] OnBehalfOf claims take second duration (#10664)

OnBehalfOf claims take second duration

Signed-off-by: Stephen Crawford 
---
 CHANGELOG.md                                  |  2 +-
 .../opensearch/identity/tokens/AuthToken.java |  1 +
 .../identity/tokens/OnBehalfOfClaims.java     | 55 +++----------------
 3 files changed, 9 insertions(+), 49 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0742215a2dfb9..f9724033f540a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
-- Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
+- Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679), [#10664](https://github.com/opensearch-project/OpenSearch/pull/10664))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
 - [Admission control] Add enhancements to FS stats to include read/write time, queue size and IO time ([#10541](https://github.com/opensearch-project/OpenSearch/pull/10541))
 - [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
diff --git a/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java b/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java
index c929e7421b3d8..88bb855a6e70d 100644
--- a/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java
+++ b/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java
@@ -16,4 +16,5 @@
 public interface AuthToken {
 
     String asAuthHeaderValue();
+
 }
diff --git a/server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java b/server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java
index 3fef248ee6d3a..00e50a59e9486 100644
--- a/server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java
+++ b/server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java
@@ -14,46 +14,17 @@
 public class OnBehalfOfClaims {
 
     private final String audience;
-    private final String subject;
-    private final Long expiration;
-    private final Long not_before;
-    private final Long issued_at;
+    private final Long expiration_seconds;
 
     /**
      * Constructor for OnBehalfOfClaims
      * @param aud the Audience for the token
-     * @param subject the subject of the token
-     * @param expiration the expiration time in seconds for the token
-     * @param not_before the not_before time in seconds for the token
-     * @param issued_at the issued_at time in seconds for the token
-     */
-    public OnBehalfOfClaims(String aud, String subject, Long expiration, Long not_before, Long issued_at) {
-        this.audience = aud;
-        this.subject = subject;
-        this.expiration = expiration;
-        this.not_before = not_before;
-        this.issued_at = issued_at;
-    }
-
-    /**
-     * A constructor that sets a default issued at time of the current time
-     * @param aud the Audience for the token
-     * @param subject the subject of the token
-     * @param expiration the expiration time in seconds for the token
-     * @param not_before the not_before time in seconds for the token
-     */
-    public OnBehalfOfClaims(String aud, String subject, Long expiration, Long not_before) {
-        this(aud, subject, expiration, not_before, System.currentTimeMillis() / 1000);
-    }
+     * @param expiration_seconds the length of time in seconds the token is valid
 
-    /**
-     * A constructor which sets a default not before time of the current time
-     * @param aud the Audience for the token
-     * @param subject the subject of the token
-     * @param expiration the expiration time in seconds for the token
      */
-    public OnBehalfOfClaims(String aud, String subject, Long expiration) {
-        this(aud, subject, expiration, System.currentTimeMillis() / 1000);
+    public OnBehalfOfClaims(String aud, Long expiration_seconds) {
+        this.audience = aud;
+        this.expiration_seconds = expiration_seconds;
     }
 
     /**
@@ -62,26 +33,14 @@ public OnBehalfOfClaims(String aud, String subject, Long expiration) {
      * @param subject the subject of the token
      */
     public OnBehalfOfClaims(String aud, String subject) {
-        this(aud, subject, System.currentTimeMillis() / 1000 + 300);
+        this(aud, 300L);
     }
 
     public String getAudience() {
         return audience;
     }
 
-    public String getSubject() {
-        return subject;
-    }
-
     public Long getExpiration() {
-        return expiration;
-    }
-
-    public Long getNot_before() {
-        return not_before;
-    }
-
-    public Long getIssued_at() {
-        return issued_at;
+        return expiration_seconds;
     }
 }

From c00141770f1cde3a26ab3e514866ad136379c7f0 Mon Sep 17 00:00:00 2001
From: Kunal Kotwani 
Date: Wed, 18 Oct 2023 08:12:53 -0700
Subject: [PATCH 181/388] Fix download reporting for segment replication
 (#10644)

Signed-off-by: Kunal Kotwani 
---
 .../PrimaryShardReplicationSource.java        |  6 ++
 .../RemoteStoreReplicationSource.java         |  9 ++-
 .../replication/SegmentReplicationSource.java | 73 +++++++++++++++++++
 .../replication/SegmentReplicationTarget.java | 23 +++++-
 .../index/shard/RemoteIndexShardTests.java    |  4 +-
 .../SegmentReplicationIndexShardTests.java    |  2 +
 ...licationWithNodeToNodeIndexShardTests.java |  6 ++
 .../PrimaryShardReplicationSourceTests.java   |  3 +
 .../RemoteStoreReplicationSourceTests.java    | 20 ++++-
 .../SegmentReplicationTargetServiceTests.java |  4 +
 .../SegmentReplicationTargetTests.java        | 50 +++++++++++++
 .../replication/TestReplicationSource.java    |  2 +
 .../index/shard/IndexShardTestCase.java       |  2 +
 13 files changed, 198 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java
index 9dcd16c53e6f3..02fc8feefd698 100644
--- a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java
+++ b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java
@@ -22,6 +22,7 @@
 import org.opensearch.transport.TransportService;
 
 import java.util.List;
+import java.util.function.BiConsumer;
 
 import static org.opensearch.indices.replication.SegmentReplicationSourceService.Actions.GET_CHECKPOINT_INFO;
 import static org.opensearch.indices.replication.SegmentReplicationSourceService.Actions.GET_SEGMENT_FILES;
@@ -80,8 +81,13 @@ public void getSegmentFiles(
         ReplicationCheckpoint checkpoint,
         List filesToFetch,
         IndexShard indexShard,
+        BiConsumer fileProgressTracker,
         ActionListener listener
     ) {
+        // fileProgressTracker is a no-op for node to node recovery
+        // MultiFileWriter takes care of progress tracking for downloads in this scenario
+        // TODO: Move state management and tracking into replication methods and use chunking and data
+        // copy mechanisms only from MultiFileWriter
         final Writeable.Reader reader = GetSegmentFilesResponse::new;
         final ActionListener responseListener = ActionListener.map(listener, r -> r);
         final GetSegmentFilesRequest request = new GetSegmentFilesRequest(
diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java
index d2000a56401f5..12eabf1e6554f 100644
--- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java
+++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java
@@ -29,6 +29,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
 /**
@@ -95,6 +96,7 @@ public void getSegmentFiles(
         ReplicationCheckpoint checkpoint,
         List filesToFetch,
         IndexShard indexShard,
+        BiConsumer fileProgressTracker,
         ActionListener listener
     ) {
         try {
@@ -117,7 +119,12 @@ public void getSegmentFiles(
                         assert directoryFiles.contains(file) == false : "Local store already contains the file " + file;
                         toDownloadSegmentNames.add(file);
                     }
-                    indexShard.getFileDownloader().download(remoteDirectory, storeDirectory, toDownloadSegmentNames);
+                    indexShard.getFileDownloader()
+                        .download(
+                            remoteDirectory,
+                            new ReplicationStatsDirectoryWrapper(storeDirectory, fileProgressTracker),
+                            toDownloadSegmentNames
+                        );
                     logger.debug("Downloaded segment files from remote store {}", filesToFetch);
                 } finally {
                     indexShard.store().decRef();
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java
index 6676b5b667e42..24f0cb15ddb25 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java
@@ -8,13 +8,19 @@
 
 package org.opensearch.indices.replication;
 
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
 import org.opensearch.common.util.CancellableThreads.ExecutionCancelledException;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.store.StoreFileMetadata;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.function.BiConsumer;
 
 /**
  * Represents the source of a replication event.
@@ -39,6 +45,7 @@ public interface SegmentReplicationSource {
      * @param checkpoint    {@link ReplicationCheckpoint} Checkpoint to fetch metadata for.
      * @param filesToFetch  {@link List} List of files to fetch.
      * @param indexShard    {@link IndexShard} Reference to the IndexShard.
+     * @param fileProgressTracker {@link BiConsumer} A consumer that updates the replication progress for shard files.
      * @param listener      {@link ActionListener} Listener that completes with the list of files copied.
      */
     void getSegmentFiles(
@@ -46,6 +53,7 @@ void getSegmentFiles(
         ReplicationCheckpoint checkpoint,
         List filesToFetch,
         IndexShard indexShard,
+        BiConsumer fileProgressTracker,
         ActionListener listener
     );
 
@@ -58,4 +66,69 @@ void getSegmentFiles(
      * Cancel any ongoing requests, should resolve any ongoing listeners with onFailure with a {@link ExecutionCancelledException}.
      */
     default void cancel() {}
+
+    /**
+     * Directory wrapper that records copy process for replication statistics
+     *
+     * @opensearch.internal
+     */
+    final class ReplicationStatsDirectoryWrapper extends FilterDirectory {
+        private final BiConsumer fileProgressTracker;
+
+        ReplicationStatsDirectoryWrapper(Directory in, BiConsumer fileProgressTracker) {
+            super(in);
+            this.fileProgressTracker = fileProgressTracker;
+        }
+
+        @Override
+        public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
+            // here we wrap the index input form the source directory to report progress of file copy for the recovery stats.
+            // we increment the num bytes recovered in the readBytes method below, if users pull statistics they can see immediately
+            // how much has been recovered.
+            in.copyFrom(new FilterDirectory(from) {
+                @Override
+                public IndexInput openInput(String name, IOContext context) throws IOException {
+                    final IndexInput input = in.openInput(name, context);
+                    return new IndexInput("StatsDirectoryWrapper(" + input.toString() + ")") {
+                        @Override
+                        public void close() throws IOException {
+                            input.close();
+                        }
+
+                        @Override
+                        public long getFilePointer() {
+                            throw new UnsupportedOperationException("only straight copies are supported");
+                        }
+
+                        @Override
+                        public void seek(long pos) throws IOException {
+                            throw new UnsupportedOperationException("seeks are not supported");
+                        }
+
+                        @Override
+                        public long length() {
+                            return input.length();
+                        }
+
+                        @Override
+                        public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+                            throw new UnsupportedOperationException("slices are not supported");
+                        }
+
+                        @Override
+                        public byte readByte() throws IOException {
+                            throw new UnsupportedOperationException("use a buffer if you wanna perform well");
+                        }
+
+                        @Override
+                        public void readBytes(byte[] b, int offset, int len) throws IOException {
+                            // we rely on the fact that copyFrom uses a buffer
+                            input.readBytes(b, offset, len);
+                            fileProgressTracker.accept(dest, (long) len);
+                        }
+                    };
+                }
+            }, src, dest, context);
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java
index 0eb6ce36fa63d..cd6dbe8af90d9 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java
@@ -170,7 +170,14 @@ public void startReplication(ActionListener listener) {
             final List filesToFetch = getFiles(checkpointInfo);
             state.setStage(SegmentReplicationState.Stage.GET_FILES);
             cancellableThreads.checkForCancel();
-            source.getSegmentFiles(getId(), checkpointInfo.getCheckpoint(), filesToFetch, indexShard, getFilesListener);
+            source.getSegmentFiles(
+                getId(),
+                checkpointInfo.getCheckpoint(),
+                filesToFetch,
+                indexShard,
+                this::updateFileRecoveryBytes,
+                getFilesListener
+            );
         }, listener::onFailure);
 
         getFilesListener.whenComplete(response -> {
@@ -240,6 +247,20 @@ private boolean validateLocalChecksum(StoreFileMetadata file) {
         }
     }
 
+    /**
+     * Updates the state to reflect recovery progress for the given file and
+     * updates the last access time for the target.
+     * @param fileName Name of the file being downloaded
+     * @param bytesRecovered Number of bytes recovered
+     */
+    private void updateFileRecoveryBytes(String fileName, long bytesRecovered) {
+        ReplicationLuceneIndex index = state.getIndex();
+        if (index != null) {
+            index.addRecoveredBytesToFile(fileName, bytesRecovered);
+        }
+        setLastAccessTime();
+    }
+
     private void finalizeReplication(CheckpointInfoResponse checkpointInfoResponse) throws OpenSearchCorruptionException {
         cancellableThreads.checkForCancel();
         state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION);
diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
index 6a99063d11353..fe389e3b3fcb4 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
@@ -43,6 +43,7 @@
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
 import static org.opensearch.index.engine.EngineTestCase.assertAtMostOneLuceneDocumentPerSequenceNumber;
@@ -388,9 +389,10 @@ public void getSegmentFiles(
                     ReplicationCheckpoint checkpoint,
                     List filesToFetch,
                     IndexShard indexShard,
+                    BiConsumer fileProgressTracker,
                     ActionListener listener
                 ) {
-                    super.getSegmentFiles(replicationId, checkpoint, filesToFetch, indexShard, listener);
+                    super.getSegmentFiles(replicationId, checkpoint, filesToFetch, indexShard, (fileName, bytesRecovered) -> {}, listener);
                     runAfterGetFiles[index.getAndIncrement()].run();
                 }
 
diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
index 528402d48658a..52f28aead533d 100644
--- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
@@ -91,6 +91,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 
 import static org.opensearch.index.engine.EngineTestCase.assertAtMostOneLuceneDocumentPerSequenceNumber;
@@ -725,6 +726,7 @@ public void getSegmentFiles(
                     ReplicationCheckpoint checkpoint,
                     List filesToFetch,
                     IndexShard indexShard,
+                    BiConsumer fileProgressTracker,
                     ActionListener listener
                 ) {
                     // set the listener, we will only fail it once we cancel the source.
diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithNodeToNodeIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithNodeToNodeIndexShardTests.java
index c394101697b47..f0950fe5392de 100644
--- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithNodeToNodeIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationWithNodeToNodeIndexShardTests.java
@@ -47,6 +47,7 @@
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -87,6 +88,7 @@ public void getSegmentFiles(
                     ReplicationCheckpoint checkpoint,
                     List filesToFetch,
                     IndexShard indexShard,
+                    BiConsumer fileProgressTracker,
                     ActionListener listener
                 ) {
                     // randomly resolve the listener, indicating the source has resolved.
@@ -131,6 +133,7 @@ public void getSegmentFiles(
                     ReplicationCheckpoint checkpoint,
                     List filesToFetch,
                     IndexShard indexShard,
+                    BiConsumer fileProgressTracker,
                     ActionListener listener
                 ) {
                     Assert.fail("Should not be reached");
@@ -176,6 +179,7 @@ public void getSegmentFiles(
                     ReplicationCheckpoint checkpoint,
                     List filesToFetch,
                     IndexShard indexShard,
+                    BiConsumer fileProgressTracker,
                     ActionListener listener
                 ) {
                     Assert.fail("Unreachable");
@@ -223,6 +227,7 @@ public void getSegmentFiles(
                     ReplicationCheckpoint checkpoint,
                     List filesToFetch,
                     IndexShard indexShard,
+                    BiConsumer fileProgressTracker,
                     ActionListener listener
                 ) {}
             };
@@ -269,6 +274,7 @@ public void getSegmentFiles(
                     ReplicationCheckpoint checkpoint,
                     List filesToFetch,
                     IndexShard indexShard,
+                    BiConsumer fileProgressTracker,
                     ActionListener listener
                 ) {
                     listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList()));
diff --git a/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java
index bcacef83d190a..e4dd32e5c6f70 100644
--- a/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/PrimaryShardReplicationSourceTests.java
@@ -125,6 +125,7 @@ public void testGetSegmentFiles() {
             checkpoint,
             Arrays.asList(testMetadata),
             mock(IndexShard.class),
+            (fileName, bytesRecovered) -> {},
             mock(ActionListener.class)
         );
         CapturingTransport.CapturedRequest[] requestList = transport.getCapturedRequestsAndClear();
@@ -153,6 +154,7 @@ public void testTransportTimeoutForGetSegmentFilesAction() {
             checkpoint,
             Arrays.asList(testMetadata),
             mock(IndexShard.class),
+            (fileName, bytesRecovered) -> {},
             mock(ActionListener.class)
         );
         CapturingTransport.CapturedRequest[] requestList = transport.getCapturedRequestsAndClear();
@@ -178,6 +180,7 @@ public void testGetSegmentFiles_CancelWhileRequestOpen() throws InterruptedExcep
             checkpoint,
             Arrays.asList(testMetadata),
             mock(IndexShard.class),
+            (fileName, bytesRecovered) -> {},
             new ActionListener<>() {
                 @Override
                 public void onResponse(GetSegmentFilesResponse getSegmentFilesResponse) {
diff --git a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java
index 9204f48ba5bdd..287962b158c79 100644
--- a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java
@@ -90,7 +90,7 @@ public void testGetSegmentFiles() throws ExecutionException, InterruptedExceptio
         List filesToFetch = primaryShard.getSegmentMetadataMap().values().stream().collect(Collectors.toList());
         final PlainActionFuture res = PlainActionFuture.newFuture();
         replicationSource = new RemoteStoreReplicationSource(primaryShard);
-        replicationSource.getSegmentFiles(REPLICATION_ID, checkpoint, filesToFetch, replicaShard, res);
+        replicationSource.getSegmentFiles(REPLICATION_ID, checkpoint, filesToFetch, replicaShard, (fileName, bytesRecovered) -> {}, res);
         GetSegmentFilesResponse response = res.get();
         assertEquals(response.files.size(), filesToFetch.size());
         assertTrue(response.files.containsAll(filesToFetch));
@@ -104,7 +104,14 @@ public void testGetSegmentFilesAlreadyExists() throws IOException, InterruptedEx
         try {
             final PlainActionFuture res = PlainActionFuture.newFuture();
             replicationSource = new RemoteStoreReplicationSource(primaryShard);
-            replicationSource.getSegmentFiles(REPLICATION_ID, checkpoint, filesToFetch, primaryShard, res);
+            replicationSource.getSegmentFiles(
+                REPLICATION_ID,
+                checkpoint,
+                filesToFetch,
+                primaryShard,
+                (fileName, bytesRecovered) -> {},
+                res
+            );
             res.get();
         } catch (AssertionError | ExecutionException ex) {
             latch.countDown();
@@ -118,7 +125,14 @@ public void testGetSegmentFilesReturnEmptyResponse() throws ExecutionException,
         final ReplicationCheckpoint checkpoint = primaryShard.getLatestReplicationCheckpoint();
         final PlainActionFuture res = PlainActionFuture.newFuture();
         replicationSource = new RemoteStoreReplicationSource(primaryShard);
-        replicationSource.getSegmentFiles(REPLICATION_ID, checkpoint, Collections.emptyList(), primaryShard, res);
+        replicationSource.getSegmentFiles(
+            REPLICATION_ID,
+            checkpoint,
+            Collections.emptyList(),
+            primaryShard,
+            (fileName, bytesRecovered) -> {},
+            res
+        );
         GetSegmentFilesResponse response = res.get();
         assert (response.files.isEmpty());
     }
diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
index d16a188f405e3..7b02635525264 100644
--- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
@@ -53,6 +53,7 @@
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
@@ -212,6 +213,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 Assert.fail("Should not be called");
@@ -277,6 +279,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList()));
@@ -334,6 +337,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 Assert.fail("Unreachable");
diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java
index a9d7d3cdd32fc..8b4b3aff701b4 100644
--- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java
@@ -40,6 +40,7 @@
 import org.opensearch.index.store.StoreTests;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 import org.opensearch.indices.replication.common.ReplicationFailedException;
+import org.opensearch.indices.replication.common.ReplicationLuceneIndex;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.DummyShardLock;
 import org.opensearch.test.IndexSettingsModule;
@@ -53,6 +54,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.function.BiConsumer;
 
 import org.mockito.Mockito;
 
@@ -131,10 +133,12 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 assertEquals(1, filesToFetch.size());
                 assert (filesToFetch.contains(SEGMENT_FILE));
+                filesToFetch.forEach(storeFileMetadata -> fileProgressTracker.accept(storeFileMetadata.name(), storeFileMetadata.length()));
                 listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
             }
         };
@@ -149,6 +153,19 @@ public void getSegmentFiles(
             public void onResponse(Void replicationResponse) {
                 try {
                     verify(spyIndexShard, times(1)).finalizeReplication(any());
+                    assertEquals(
+                        1,
+                        segrepTarget.state()
+                            .getIndex()
+                            .fileDetails()
+                            .stream()
+                            .filter(ReplicationLuceneIndex.FileMetadata::fullyRecovered)
+                            .count()
+                    );
+                    assertEquals(
+                        0,
+                        segrepTarget.state().getIndex().fileDetails().stream().filter(file -> file.fullyRecovered() == false).count()
+                    );
                     segrepTarget.markAsDone();
                 } catch (IOException ex) {
                     Assert.fail();
@@ -182,6 +199,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
@@ -200,6 +218,15 @@ public void onResponse(Void replicationResponse) {
 
             @Override
             public void onFailure(Exception e) {
+                assertEquals(
+                    0,
+                    segrepTarget.state()
+                        .getIndex()
+                        .fileDetails()
+                        .stream()
+                        .filter(ReplicationLuceneIndex.FileMetadata::fullyRecovered)
+                        .count()
+                );
                 assertEquals(exception, e.getCause().getCause());
                 segrepTarget.fail(new ReplicationFailedException(e), false);
             }
@@ -225,6 +252,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 listener.onFailure(exception);
@@ -243,6 +271,15 @@ public void onResponse(Void replicationResponse) {
 
             @Override
             public void onFailure(Exception e) {
+                assertEquals(
+                    0,
+                    segrepTarget.state()
+                        .getIndex()
+                        .fileDetails()
+                        .stream()
+                        .filter(ReplicationLuceneIndex.FileMetadata::fullyRecovered)
+                        .count()
+                );
                 assertEquals(exception, e.getCause().getCause());
                 segrepTarget.fail(new ReplicationFailedException(e), false);
             }
@@ -268,6 +305,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
@@ -314,6 +352,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
@@ -358,6 +397,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
@@ -376,6 +416,15 @@ public void onResponse(Void replicationResponse) {
 
             @Override
             public void onFailure(Exception e) {
+                assertEquals(
+                    0,
+                    segrepTarget.state()
+                        .getIndex()
+                        .fileDetails()
+                        .stream()
+                        .filter(ReplicationLuceneIndex.FileMetadata::fullyRecovered)
+                        .count()
+                );
                 assertTrue(e instanceof OpenSearchCorruptionException);
                 assertTrue(e.getMessage().contains("has local copies of segments that differ from the primary"));
                 segrepTarget.fail(new ReplicationFailedException(e), false);
@@ -410,6 +459,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
diff --git a/test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java b/test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java
index b29e25a0bff2c..bcd47e3d578ee 100644
--- a/test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java
+++ b/test/framework/src/main/java/org/opensearch/index/replication/TestReplicationSource.java
@@ -17,6 +17,7 @@
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 
 import java.util.List;
+import java.util.function.BiConsumer;
 
 /**
  * This class is used by unit tests implementing SegmentReplicationSource
@@ -36,6 +37,7 @@ public abstract void getSegmentFiles(
         ReplicationCheckpoint checkpoint,
         List filesToFetch,
         IndexShard indexShard,
+        BiConsumer fileProgressTracker,
         ActionListener listener
     );
 
diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
index 9dc230474482f..412d5235fe462 100644
--- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
@@ -174,6 +174,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -1620,6 +1621,7 @@ public void getSegmentFiles(
                 ReplicationCheckpoint checkpoint,
                 List filesToFetch,
                 IndexShard indexShard,
+                BiConsumer fileProgressTracker,
                 ActionListener listener
             ) {
                 try (

From 3a36c2256e838fc0ce04b5c144257e3096ca29b7 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Wed, 18 Oct 2023 20:46:20 +0530
Subject: [PATCH 182/388] Download functionality of global metadata from remote
 store (#10535)

* Download funcationality of global metadata from remote store

Signed-off-by: Dhwanil Patel 
---
 CHANGELOG.md                                  |   1 +
 .../remote/RemoteClusterStateServiceIT.java   |   4 +-
 .../remote/RemoteClusterStateService.java     |  64 ++++++++--
 .../recovery/RemoteStoreRestoreService.java   |   3 +-
 .../RemoteClusterStateServiceTests.java       | 115 ++++++++++++++++--
 5 files changed, 168 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f9724033f540a..872cf022671cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
 - [Remote cluster state] Change file names for remote cluster state ([#10557](https://github.com/opensearch-project/OpenSearch/pull/10557))
 - [Remote cluster state] Upload global metadata in cluster state to remote store([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
+- [Remote cluster state] Download functionality of global metadata from remote store ([#10535](https://github.com/opensearch-project/OpenSearch/pull/10535))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
index 6fcc89cfe9e9a..7304304e522f8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
@@ -86,10 +86,10 @@ public void testFullClusterRestoreStaleDelete() throws Exception {
 
         assertEquals(10, repository.blobStore().blobContainer(baseMetadataPath.add("manifest")).listBlobsByPrefix("manifest").size());
 
-        Map indexMetadataMap = remoteClusterStateService.getLatestIndexMetadata(
+        Map indexMetadataMap = remoteClusterStateService.getLatestMetadata(
             cluster().getClusterName(),
             getClusterState().metadata().clusterUUID()
-        );
+        ).getIndices();
         assertEquals(0, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas());
         assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards());
     }
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 2092c2a0aac3f..358ce600a49d8 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -662,18 +662,18 @@ private BlobPath getManifestFolderPath(String clusterName, String clusterUUID) {
      *
      * @param clusterUUID uuid of cluster state to refer to in remote
      * @param clusterName name of the cluster
+     * @param clusterMetadataManifest manifest file of cluster
      * @return {@code Map} latest IndexUUID to IndexMetadata map
      */
-    public Map getLatestIndexMetadata(String clusterName, String clusterUUID) throws IOException {
-        start();
-        Map remoteIndexMetadata = new HashMap<>();
-        Optional clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
-        if (!clusterMetadataManifest.isPresent()) {
-            throw new IllegalStateException("Latest index metadata is not present for the provided clusterUUID");
-        }
-        assert Objects.equals(clusterUUID, clusterMetadataManifest.get().getClusterUUID())
+    private Map getIndexMetadataMap(
+        String clusterName,
+        String clusterUUID,
+        ClusterMetadataManifest clusterMetadataManifest
+    ) {
+        assert Objects.equals(clusterUUID, clusterMetadataManifest.getClusterUUID())
             : "Corrupt ClusterMetadataManifest found. Cluster UUID mismatch.";
-        for (UploadedIndexMetadata uploadedIndexMetadata : clusterMetadataManifest.get().getIndices()) {
+        Map remoteIndexMetadata = new HashMap<>();
+        for (UploadedIndexMetadata uploadedIndexMetadata : clusterMetadataManifest.getIndices()) {
             IndexMetadata indexMetadata = getIndexMetadata(clusterName, clusterUUID, uploadedIndexMetadata);
             remoteIndexMetadata.put(uploadedIndexMetadata.getIndexUUID(), indexMetadata);
         }
@@ -704,6 +704,52 @@ private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, U
         }
     }
 
+    /**
+     * Fetch latest metadata from remote cluster state including global metadata and index metadata
+     *
+     * @param clusterUUID uuid of cluster state to refer to in remote
+     * @param clusterName name of the cluster
+     * @return {@link IndexMetadata}
+     */
+    public Metadata getLatestMetadata(String clusterName, String clusterUUID) throws IOException {
+        start();
+        Optional clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
+        if (!clusterMetadataManifest.isPresent()) {
+            throw new IllegalStateException(
+                String.format(Locale.ROOT, "Latest cluster metadata manifest is not present for the provided clusterUUID: %s", clusterUUID)
+            );
+        }
+        // Fetch Global Metadata
+        Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get());
+
+        // Fetch Index Metadata
+        Map indices = getIndexMetadataMap(clusterName, clusterUUID, clusterMetadataManifest.get());
+
+        return Metadata.builder(globalMetadata).indices(indices).build();
+    }
+
+    private Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) {
+        String globalMetadataFileName = clusterMetadataManifest.getGlobalMetadataFileName();
+        try {
+            // Fetch Global metadata
+            if (globalMetadataFileName != null) {
+                String[] splitPath = globalMetadataFileName.split("/");
+                return GLOBAL_METADATA_FORMAT.read(
+                    globalMetadataContainer(clusterName, clusterUUID),
+                    splitPath[splitPath.length - 1],
+                    blobStoreRepository.getNamedXContentRegistry()
+                );
+            } else {
+                return Metadata.EMPTY_METADATA;
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException(
+                String.format(Locale.ROOT, "Error while downloading Global Metadata - %s", globalMetadataFileName),
+                e
+            );
+        }
+    }
+
     /**
      * Fetch latest ClusterMetadataManifest from remote state store
      *
diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
index 94fd08b99ac58..dec999e43110f 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
@@ -141,7 +141,8 @@ public RemoteRestoreResult restore(
             || restoreClusterUUID.isBlank()) == false;
         if (metadataFromRemoteStore) {
             try {
-                remoteClusterStateService.getLatestIndexMetadata(currentState.getClusterName().value(), restoreClusterUUID)
+                remoteClusterStateService.getLatestMetadata(currentState.getClusterName().value(), restoreClusterUUID)
+                    .getIndices()
                     .values()
                     .forEach(indexMetadata -> {
                         indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index ddee3bcf10382..49b7f0ff8d1a9 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -12,6 +12,7 @@
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.CoordinationMetadata;
+import org.opensearch.cluster.metadata.IndexGraveyard;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNodes;
@@ -28,6 +29,7 @@
 import org.opensearch.common.lucene.store.ByteArrayIndexInput;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -634,7 +636,8 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE
 
         remoteClusterStateService.start();
         assertEquals(
-            remoteClusterStateService.getLatestIndexMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+            remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+                .getIndices()
                 .size(),
             0
         );
@@ -662,10 +665,8 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
         remoteClusterStateService.start();
         Exception e = assertThrows(
             IllegalStateException.class,
-            () -> remoteClusterStateService.getLatestIndexMetadata(
-                clusterState.getClusterName().value(),
-                clusterState.metadata().clusterUUID()
-            )
+            () -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+                .getIndices()
         );
         assertEquals(e.getMessage(), "Error while downloading IndexMetadata - " + uploadedIndexMetadata.getUploadedFilename());
     }
@@ -704,6 +705,70 @@ public void testReadLatestMetadataManifestSuccess() throws IOException {
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
     }
 
+    public void testReadGlobalMetadata() throws IOException {
+        when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(new NamedXContentRegistry(
+            List.of(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(IndexGraveyard.TYPE), IndexGraveyard::fromXContent))));
+        final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build();
+        remoteClusterStateService.start();
+
+        final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
+            .indices(List.of())
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .stateUUID("state-uuid")
+            .clusterUUID("cluster-uuid")
+            .codecVersion(MANIFEST_CURRENT_CODEC_VERSION)
+            .globalMetadataFileName("global-metadata-file")
+            .nodeId("nodeA")
+            .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
+            .previousClusterUUID("prev-cluster-uuid")
+            .build();
+
+        Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build();
+        mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expactedMetadata);
+
+        Metadata metadata = remoteClusterStateService.getLatestMetadata(
+            clusterState.getClusterName().value(),
+            clusterState.metadata().clusterUUID()
+        );
+
+        assertTrue(Metadata.isGlobalStateEquals(metadata, expactedMetadata));
+    }
+
+    public void testReadGlobalMetadataIOException() throws IOException {
+        final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build();
+        remoteClusterStateService.start();
+        String globalIndexMetadataName = "global-metadata-file";
+        final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
+            .indices(List.of())
+            .clusterTerm(1L)
+            .stateVersion(1L)
+            .stateUUID("state-uuid")
+            .clusterUUID("cluster-uuid")
+            .codecVersion(MANIFEST_CURRENT_CODEC_VERSION)
+            .globalMetadataFileName(globalIndexMetadataName)
+            .nodeId("nodeA")
+            .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
+            .previousClusterUUID("prev-cluster-uuid")
+            .build();
+
+        Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build();
+
+        BlobContainer blobContainer = mockBlobStoreObjects();
+        mockBlobContainerForGlobalMetadata(blobContainer, expectedManifest, expactedMetadata);
+
+        when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(globalIndexMetadataName))).thenThrow(
+            FileNotFoundException.class
+        );
+
+        remoteClusterStateService.start();
+        Exception e = assertThrows(
+            IllegalStateException.class,
+            () -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+        );
+        assertEquals(e.getMessage(), "Error while downloading Global Metadata - " + globalIndexMetadataName);
+    }
+
     public void testReadLatestIndexMetadataSuccess() throws IOException {
         final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
         remoteClusterStateService.start();
@@ -730,15 +795,16 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
+            .globalMetadataFileName("global-metadata-file")
             .codecVersion(ClusterMetadataManifest.CODEC_V0)
             .build();
 
         mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
 
-        Map indexMetadataMap = remoteClusterStateService.getLatestIndexMetadata(
+        Map indexMetadataMap = remoteClusterStateService.getLatestMetadata(
             clusterState.getClusterName().value(),
             clusterState.metadata().clusterUUID()
-        );
+        ).getIndices();
 
         assertEquals(indexMetadataMap.size(), 1);
         assertEquals(indexMetadataMap.get(index.getUUID()).getIndex().getName(), index.getName());
@@ -1114,6 +1180,41 @@ private void mockBlobContainer(
         });
     }
 
+    private void mockBlobContainerForGlobalMetadata(
+        BlobContainer blobContainer,
+        ClusterMetadataManifest clusterMetadataManifest,
+        Metadata metadata
+    ) throws IOException {
+        String mockManifestFileName = "manifest__1__2__C__456__1";
+        BlobMetadata blobMetadata = new PlainBlobMetadata(mockManifestFileName, 1);
+        when(
+            blobContainer.listBlobsByPrefixInSortedOrder(
+                "manifest" + RemoteClusterStateService.DELIMITER,
+                1,
+                BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC
+            )
+        ).thenReturn(Arrays.asList(blobMetadata));
+
+        BytesReference bytes = RemoteClusterStateService.CLUSTER_METADATA_MANIFEST_FORMAT.serialize(
+            clusterMetadataManifest,
+            mockManifestFileName,
+            blobStoreRepository.getCompressor(),
+            FORMAT_PARAMS
+        );
+        when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
+
+        BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize(
+            metadata,
+            "global-metadata-file",
+            blobStoreRepository.getCompressor(),
+            FORMAT_PARAMS
+        );
+        String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/");
+        when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))).thenReturn(
+            new ByteArrayInputStream(bytesGlobalMetadata.streamInput().readAllBytes())
+        );
+    }
+
     private static ClusterState.Builder generateClusterStateWithGlobalMetadata() {
         final Settings clusterSettings = Settings.builder().put("cluster.blocks.read_only", true).build();
         final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();

From 56736be1adaa0c7e97d8db159ff6dca6f6814a9e Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Wed, 18 Oct 2023 23:11:53 +0530
Subject: [PATCH 183/388] Remote Restore Cluster Metadata if local disk state
 lost after quorum loss recovery (#10447)

* Remote Restore Cluster Metadata if local disk state lost after quorum loss

Signed-off-by: bansvaru 
---
 CHANGELOG.md                                  |   4 +
 .../remotestore/BaseRemoteStoreRestoreIT.java |   8 +-
 .../RemoteStoreClusterStateRestoreIT.java     | 137 ++++++++++++++++++
 .../opensearch/gateway/GatewayMetaState.java  |  58 ++++----
 .../remote/RemoteClusterStateService.java     |   7 +-
 .../recovery/RemoteStoreRestoreService.java   |  58 ++++++--
 .../GatewayMetaStatePersistedStateTests.java  |  83 ++++++++++-
 .../gateway/MockGatewayMetaState.java         |  25 +++-
 8 files changed, 330 insertions(+), 50 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 872cf022671cb..f4332c7847db6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,7 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
 - [Remote cluster state] Change file names for remote cluster state ([#10557](https://github.com/opensearch-project/OpenSearch/pull/10557))
 - [Remote cluster state] Upload global metadata in cluster state to remote store([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
+<<<<<<< Updated upstream
 - [Remote cluster state] Download functionality of global metadata from remote store ([#10535](https://github.com/opensearch-project/OpenSearch/pull/10535))
+=======
+- [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
+>>>>>>> Stashed changes
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java
index ad3e99dd274ce..b8481610869e6 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java
@@ -58,7 +58,7 @@ protected void restore(String... indices) {
             );
     }
 
-    protected void verifyRestoredData(Map indexStats, String indexName) throws Exception {
+    protected void verifyRestoredData(Map indexStats, String indexName, boolean indexMoreData) throws Exception {
         ensureYellowAndNoInitializingShards(indexName);
         ensureGreen(indexName);
         // This is to ensure that shards that were already assigned will get latest count
@@ -68,6 +68,8 @@ protected void verifyRestoredData(Map indexStats, String indexName
             30,
             TimeUnit.SECONDS
         );
+        if (indexMoreData == false) return;
+
         IndexResponse response = indexSingleDoc(indexName);
         if (indexStats.containsKey(MAX_SEQ_NO_TOTAL + "-shard-" + response.getShardId().id())) {
             assertEquals(indexStats.get(MAX_SEQ_NO_TOTAL + "-shard-" + response.getShardId().id()) + 1, response.getSeqNo());
@@ -80,6 +82,10 @@ protected void verifyRestoredData(Map indexStats, String indexName
         );
     }
 
+    protected void verifyRestoredData(Map indexStats, String indexName) throws Exception {
+        verifyRestoredData(indexStats, indexName, true);
+    }
+
     public void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) {
         prepareCluster(numClusterManagerNodes, numDataOnlyNodes, indices, replicaCount, shardCount, Settings.EMPTY);
     }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index 3df4cc4e34d93..3a3e293de9b13 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -8,9 +8,15 @@
 
 package org.opensearch.remotestore;
 
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.block.ClusterBlockException;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.IndexTemplateMetadata;
 import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
@@ -19,11 +25,19 @@
 
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ExecutionException;
 
+import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING;
+import static org.opensearch.cluster.metadata.Metadata.CLUSTER_READ_ONLY_BLOCK;
+import static org.opensearch.cluster.metadata.Metadata.SETTING_READ_ONLY_SETTING;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
+import static org.opensearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE;
+import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreClusterStateRestoreIT extends BaseRemoteStoreRestoreIT {
@@ -86,6 +100,7 @@ public void testFullClusterRestoreMultipleIndices() throws Exception {
         Map indexStats2 = indexData(1, false, secondIndexName);
         assertEquals((shardCount + 1) * (replicaCount + 1), getNumShards(secondIndexName).totalNumShards);
         ensureGreen(secondIndexName);
+        updateIndexBlock(true, secondIndexName);
 
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
 
@@ -98,6 +113,13 @@ public void testFullClusterRestoreMultipleIndices() throws Exception {
         // Step - 3 Trigger full cluster restore
         validateMetadata(List.of(INDEX_NAME, secondIndexName));
         verifyRestoredData(indexStats, INDEX_NAME);
+        verifyRestoredData(indexStats2, secondIndexName, false);
+        assertTrue(INDEX_READ_ONLY_SETTING.get(clusterService().state().metadata().index(secondIndexName).getSettings()));
+        assertThrows(ClusterBlockException.class, () -> indexSingleDoc(secondIndexName));
+        // Test is complete
+
+        // Remove the block to ensure proper cleanup
+        updateIndexBlock(false, secondIndexName);
     }
 
     public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathThrowsException() throws Exception {
@@ -192,4 +214,119 @@ private void validateCurrentMetadata() throws Exception {
             }
         });
     }
+
+    public void testFullClusterRestoreGlobalMetadata() throws Exception {
+        int shardCount = randomIntBetween(1, 2);
+        int replicaCount = 1;
+        int dataNodeCount = shardCount * (replicaCount + 1);
+        int clusterManagerNodeCount = 1;
+
+        // Step - 1 index some data to generate files in remote directory
+        Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, 1);
+        String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+
+        // Create global metadata - register a custom repo
+        // TODO - uncomment after all customs is also uploaded for all repos - https://github.com/opensearch-project/OpenSearch/issues/10691
+        // registerCustomRepository();
+
+        // Create global metadata - persistent settings
+        updatePersistentSettings(Settings.builder().put(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey(), 34).build());
+
+        // Create global metadata - index template
+        putIndexTemplate();
+
+        // Create global metadata - Put cluster block
+        addClusterLevelReadOnlyBlock();
+
+        // Step - 2 Replace all nodes in the cluster with new nodes. This ensures new cluster state doesn't have previous index metadata
+        resetCluster(dataNodeCount, clusterManagerNodeCount);
+
+        String newClusterUUID = clusterService().state().metadata().clusterUUID();
+        assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
+
+        // Step - 3 Trigger full cluster restore and validate
+        // validateCurrentMetadata();
+        verifyRestoredData(indexStats, INDEX_NAME, false);
+
+        // validate global metadata restored
+        verifyRestoredRepositories();
+        verifyRestoredIndexTemplate();
+        assertEquals(Integer.valueOf(34), SETTING_CLUSTER_MAX_SHARDS_PER_NODE.get(clusterService().state().metadata().settings()));
+        assertEquals(true, SETTING_READ_ONLY_SETTING.get(clusterService().state().metadata().settings()));
+        assertTrue(clusterService().state().blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK));
+        // Test is complete
+
+        // Remote the cluster read only block to ensure proper cleanup
+        updatePersistentSettings(Settings.builder().put(SETTING_READ_ONLY_SETTING.getKey(), false).build());
+        assertFalse(clusterService().state().blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK));
+    }
+
+    private void registerCustomRepository() {
+        assertAcked(
+            client().admin()
+                .cluster()
+                .preparePutRepository("custom-repo")
+                .setType("fs")
+                .setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", false))
+                .get()
+        );
+    }
+
+    private void verifyRestoredRepositories() {
+        RepositoriesMetadata repositoriesMetadata = clusterService().state().metadata().custom(RepositoriesMetadata.TYPE);
+        assertEquals(2, repositoriesMetadata.repositories().size()); // includes remote store repo as well
+        assertTrue(SYSTEM_REPOSITORY_SETTING.get(repositoriesMetadata.repository(REPOSITORY_NAME).settings()));
+        assertTrue(SYSTEM_REPOSITORY_SETTING.get(repositoriesMetadata.repository(REPOSITORY_2_NAME).settings()));
+        // TODO - uncomment after all customs is also uploaded for all repos - https://github.com/opensearch-project/OpenSearch/issues/10691
+        // assertEquals("fs", repositoriesMetadata.repository("custom-repo").type());
+        // assertEquals(Settings.builder().put("location", randomRepoPath()).put("compress", false).build(),
+        // repositoriesMetadata.repository("custom-repo").settings());
+    }
+
+    private void addClusterLevelReadOnlyBlock() throws InterruptedException, ExecutionException {
+        updatePersistentSettings(Settings.builder().put(SETTING_READ_ONLY_SETTING.getKey(), true).build());
+        assertTrue(clusterService().state().blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK));
+    }
+
+    private void updatePersistentSettings(Settings settings) throws ExecutionException, InterruptedException {
+        ClusterUpdateSettingsRequest resetRequest = new ClusterUpdateSettingsRequest();
+        resetRequest.persistentSettings(settings);
+        assertAcked(client().admin().cluster().updateSettings(resetRequest).get());
+    }
+
+    private void verifyRestoredIndexTemplate() {
+        Map indexTemplateMetadataMap = clusterService().state().metadata().templates();
+        assertEquals(1, indexTemplateMetadataMap.size());
+        assertEquals(Arrays.asList("pattern-1", "log-*"), indexTemplateMetadataMap.get("my-template").patterns());
+        assertEquals(
+            Settings.builder() // <1>
+                .put("index.number_of_shards", 3)
+                .put("index.number_of_replicas", 1)
+                .build(),
+            indexTemplateMetadataMap.get("my-template").settings()
+        );
+    }
+
+    private static void putIndexTemplate() {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest("my-template"); // <1>
+        request.patterns(Arrays.asList("pattern-1", "log-*")); // <2>
+
+        request.settings(
+            Settings.builder() // <1>
+                .put("index.number_of_shards", 3)
+                .put("index.number_of_replicas", 1)
+        );
+        assertTrue(client().admin().indices().putTemplate(request).actionGet().isAcknowledged());
+    }
+
+    private static void updateIndexBlock(boolean value, String secondIndexName) throws InterruptedException, ExecutionException {
+        assertAcked(
+            client().admin()
+                .indices()
+                .updateSettings(
+                    new UpdateSettingsRequest(Settings.builder().put(INDEX_READ_ONLY_SETTING.getKey(), value).build(), secondIndexName)
+                )
+                .get()
+        );
+    }
 }
diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
index d3e7a0c482ee2..9eb7fb0ca04d0 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -158,38 +158,44 @@ public void start(
                 PersistedState remotePersistedState = null;
                 boolean success = false;
                 try {
-                    ClusterState clusterState = prepareInitialClusterState(
+                    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings))
+                        .version(lastAcceptedVersion)
+                        .metadata(metadata)
+                        .build();
+
+                    if (DiscoveryNode.isClusterManagerNode(settings) && isRemoteStoreClusterStateEnabled(settings)) {
+                        // If the cluster UUID loaded from local is unknown (_na_) then fetch the best state from remote
+                        // If there is no valid state on remote, continue with initial empty state
+                        // If there is a valid state, then restore index metadata using this state
+                        String lastKnownClusterUUID = ClusterState.UNKNOWN_UUID;
+                        if (ClusterState.UNKNOWN_UUID.equals(clusterState.metadata().clusterUUID())) {
+                            lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote(
+                                clusterState.getClusterName().value()
+                            );
+                            if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) {
+                                // Load state from remote
+                                final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore(
+                                    clusterState,
+                                    lastKnownClusterUUID,
+                                    false,
+                                    new String[] {}
+                                );
+                                clusterState = remoteRestoreResult.getClusterState();
+                            }
+                        }
+                        remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID);
+                    }
+
+                    // Recovers Cluster and Index level blocks
+                    clusterState = prepareInitialClusterState(
                         transportService,
                         clusterService,
-                        ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings))
-                            .version(lastAcceptedVersion)
-                            .metadata(upgradeMetadataForNode(metadata, metadataIndexUpgradeService, metadataUpgrader))
+                        ClusterState.builder(clusterState)
+                            .metadata(upgradeMetadataForNode(clusterState.metadata(), metadataIndexUpgradeService, metadataUpgrader))
                             .build()
                     );
 
                     if (DiscoveryNode.isClusterManagerNode(settings)) {
-                        if (isRemoteStoreClusterStateEnabled(settings)) {
-                            // If the cluster UUID loaded from local is unknown (_na_) then fetch the best state from remote
-                            // If there is no valid state on remote, continue with initial empty state
-                            // If there is a valid state, then restore index metadata using this state
-                            String lastKnownClusterUUID = ClusterState.UNKNOWN_UUID;
-                            if (ClusterState.UNKNOWN_UUID.equals(clusterState.metadata().clusterUUID())) {
-                                lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote(
-                                    clusterState.getClusterName().value()
-                                );
-                                if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) {
-                                    // Load state from remote
-                                    final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore(
-                                        clusterState,
-                                        lastKnownClusterUUID,
-                                        false,
-                                        new String[] {}
-                                    );
-                                    clusterState = remoteRestoreResult.getClusterState();
-                                }
-                            }
-                            remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID);
-                        }
                         persistedState = new LucenePersistedState(persistedClusterStateService, currentTerm, clusterState);
                     } else {
                         persistedState = new AsyncLucenePersistedState(
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 358ce600a49d8..b9d06c8fbb1c1 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -711,7 +711,7 @@ private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, U
      * @param clusterName name of the cluster
      * @return {@link IndexMetadata}
      */
-    public Metadata getLatestMetadata(String clusterName, String clusterUUID) throws IOException {
+    public Metadata getLatestMetadata(String clusterName, String clusterUUID) {
         start();
         Optional clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
         if (!clusterMetadataManifest.isPresent()) {
@@ -759,10 +759,7 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust
      */
     public Optional getLatestClusterMetadataManifest(String clusterName, String clusterUUID) {
         Optional latestManifestFileName = getLatestManifestFileName(clusterName, clusterUUID);
-        if (latestManifestFileName.isPresent()) {
-            return Optional.of(fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, latestManifestFileName.get()));
-        }
-        return Optional.empty();
+        return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s));
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
index dec999e43110f..ac9cf35d1d8e5 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
@@ -10,15 +10,16 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateUpdateTask;
 import org.opensearch.cluster.block.ClusterBlocks;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.IndexTemplateMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.MetadataCreateIndexService;
 import org.opensearch.cluster.metadata.MetadataIndexUpgradeService;
+import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RoutingTable;
@@ -27,6 +28,7 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.shard.ShardId;
@@ -42,11 +44,13 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
+import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING;
 
 /**
  * Service responsible for restoring index data from remote store
@@ -136,17 +140,16 @@ public RemoteRestoreResult restore(
         String[] indexNames
     ) {
         Map> indexMetadataMap = new HashMap<>();
+        Metadata remoteMetadata = null;
         boolean metadataFromRemoteStore = (restoreClusterUUID == null
             || restoreClusterUUID.isEmpty()
             || restoreClusterUUID.isBlank()) == false;
         if (metadataFromRemoteStore) {
             try {
-                remoteClusterStateService.getLatestMetadata(currentState.getClusterName().value(), restoreClusterUUID)
-                    .getIndices()
-                    .values()
-                    .forEach(indexMetadata -> {
-                        indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));
-                    });
+                remoteMetadata = remoteClusterStateService.getLatestMetadata(currentState.getClusterName().value(), restoreClusterUUID);
+                remoteMetadata.getIndices().values().forEach(indexMetadata -> {
+                    indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));
+                });
             } catch (Exception e) {
                 throw new IllegalStateException("Unable to restore remote index metadata", e);
             }
@@ -161,7 +164,7 @@ public RemoteRestoreResult restore(
             }
         }
         validate(currentState, indexMetadataMap, restoreClusterUUID, restoreAllShards);
-        return executeRestore(currentState, indexMetadataMap, restoreAllShards);
+        return executeRestore(currentState, indexMetadataMap, restoreAllShards, remoteMetadata);
     }
 
     /**
@@ -174,7 +177,8 @@ public RemoteRestoreResult restore(
     private RemoteRestoreResult executeRestore(
         ClusterState currentState,
         Map> indexMetadataMap,
-        boolean restoreAllShards
+        boolean restoreAllShards,
+        Metadata remoteMetadata
     ) {
         final String restoreUUID = UUIDs.randomBase64UUID();
         List indicesToBeRestored = new ArrayList<>();
@@ -227,6 +231,10 @@ private RemoteRestoreResult executeRestore(
             totalShards += updatedIndexMetadata.getNumberOfShards();
         }
 
+        if (remoteMetadata != null) {
+            restoreGlobalMetadata(mdBuilder, remoteMetadata);
+        }
+
         RestoreInfo restoreInfo = new RestoreInfo("remote_store", indicesToBeRestored, totalShards, totalShards);
 
         RoutingTable rt = rtBuilder.build();
@@ -234,6 +242,36 @@ private RemoteRestoreResult executeRestore(
         return RemoteRestoreResult.build(restoreUUID, restoreInfo, allocationService.reroute(updatedState, "restored from remote store"));
     }
 
+    private void restoreGlobalMetadata(Metadata.Builder mdBuilder, Metadata remoteMetadata) {
+        if (remoteMetadata.persistentSettings() != null) {
+            Settings settings = remoteMetadata.persistentSettings();
+            clusterService.getClusterSettings().validateUpdate(settings);
+            mdBuilder.persistentSettings(settings);
+        }
+        if (remoteMetadata.templates() != null) {
+            for (final IndexTemplateMetadata cursor : remoteMetadata.templates().values()) {
+                mdBuilder.put(cursor);
+            }
+        }
+        if (remoteMetadata.customs() != null) {
+            for (final Map.Entry cursor : remoteMetadata.customs().entrySet()) {
+                if (RepositoriesMetadata.TYPE.equals(cursor.getKey()) == false) {
+                    mdBuilder.putCustom(cursor.getKey(), cursor.getValue());
+                }
+            }
+        }
+        Optional repositoriesMetadata = Optional.ofNullable(remoteMetadata.custom(RepositoriesMetadata.TYPE));
+        repositoriesMetadata = repositoriesMetadata.map(
+            repositoriesMetadata1 -> new RepositoriesMetadata(
+                repositoriesMetadata1.repositories()
+                    .stream()
+                    .filter(repository -> SYSTEM_REPOSITORY_SETTING.get(repository.settings()) == false)
+                    .collect(Collectors.toList())
+            )
+        );
+        repositoriesMetadata.ifPresent(metadata -> mdBuilder.putCustom(RepositoriesMetadata.TYPE, metadata));
+    }
+
     /**
      * Performs various validations needed before executing restore
      * @param currentState current cluster state
@@ -298,8 +336,6 @@ private void validate(
                         throw new IllegalStateException(finalErrorMsg);
                     }
 
-                    Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion();
-                    metadataIndexUpgradeService.upgradeIndexMetadata(indexMetadata, minIndexCompatibilityVersion);
                     boolean isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(indexMetadata.getSettings());
                     createIndexService.validateIndexName(indexName, currentState);
                     createIndexService.validateDotIndex(indexName, isHidden);
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
index c7ed1cb732154..1d5c2a0f01b5c 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
@@ -840,7 +840,8 @@ public void testGatewayForRemoteStateForInitialBootstrap() throws IOException {
                 remoteClusterStateService,
                 remoteStoreRestoreService,
                 persistedStateRegistry,
-                ClusterState.EMPTY_STATE
+                ClusterState.EMPTY_STATE,
+                false
             );
             final CoordinationState.PersistedState lucenePersistedState = gateway.getPersistedState();
             PersistedState remotePersistedState = persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE);
@@ -886,7 +887,8 @@ public void testGatewayForRemoteStateForNodeReplacement() throws IOException {
                 remoteClusterStateService,
                 remoteStoreRestoreService,
                 persistedStateRegistry,
-                ClusterState.EMPTY_STATE
+                ClusterState.EMPTY_STATE,
+                false
             );
             final CoordinationState.PersistedState lucenePersistedState = gateway.getPersistedState();
             PersistedState remotePersistedState = persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE);
@@ -918,7 +920,13 @@ public void testGatewayForRemoteStateForNodeReboot() throws IOException {
                     .clusterUUID(randomAlphaOfLength(10))
                     .build()
             );
-            gateway = newGatewayForRemoteState(remoteClusterStateService, remoteStoreRestoreService, persistedStateRegistry, clusterState);
+            gateway = newGatewayForRemoteState(
+                remoteClusterStateService,
+                remoteStoreRestoreService,
+                persistedStateRegistry,
+                clusterState,
+                false
+            );
             final CoordinationState.PersistedState lucenePersistedState = gateway.getPersistedState();
             PersistedState remotePersistedState = persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE);
             verifyNoInteractions(remoteClusterStateService);
@@ -933,13 +941,77 @@ public void testGatewayForRemoteStateForNodeReboot() throws IOException {
         }
     }
 
+    public void testGatewayForRemoteStateForInitialBootstrapBlocksApplied() throws IOException {
+        MockGatewayMetaState gateway = null;
+        try {
+            final RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class);
+            when(remoteClusterStateService.getLastKnownUUIDFromRemote(clusterName.value())).thenReturn("test-cluster-uuid");
+
+            final IndexMetadata indexMetadata = IndexMetadata.builder("test-index1")
+                .settings(
+                    settings(Version.CURRENT).put(SETTING_INDEX_UUID, randomAlphaOfLength(10))
+                        .put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true)
+                )
+                .numberOfShards(5)
+                .numberOfReplicas(1)
+                .build();
+
+            final ClusterState clusterState = ClusterState.builder(
+                createClusterState(
+                    randomNonNegativeLong(),
+                    Metadata.builder()
+                        .coordinationMetadata(CoordinationMetadata.builder().term(randomLong()).build())
+                        .put(indexMetadata, false)
+                        .clusterUUID(ClusterState.UNKNOWN_UUID)
+                        .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), true).build())
+                        .build()
+                )
+            ).nodes(DiscoveryNodes.EMPTY_NODES).build();
+
+            final RemoteStoreRestoreService remoteStoreRestoreService = mock(RemoteStoreRestoreService.class);
+            when(remoteStoreRestoreService.restore(any(), any(), anyBoolean(), any())).thenReturn(
+                RemoteRestoreResult.build("test-cluster-uuid", null, clusterState)
+            );
+            final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry();
+            gateway = newGatewayForRemoteState(
+                remoteClusterStateService,
+                remoteStoreRestoreService,
+                persistedStateRegistry,
+                ClusterState.EMPTY_STATE,
+                true
+            );
+            PersistedState remotePersistedState = persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE);
+            PersistedState lucenePersistedState = persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL);
+            verify(remoteClusterStateService).getLastKnownUUIDFromRemote(clusterName.value()); // change this
+            verify(remoteStoreRestoreService).restore(any(ClusterState.class), any(String.class), anyBoolean(), any(String[].class));
+            assertThat(remotePersistedState.getLastAcceptedState(), nullValue());
+            assertThat(
+                Metadata.isGlobalStateEquals(lucenePersistedState.getLastAcceptedState().metadata(), clusterState.metadata()),
+                equalTo(true)
+            );
+            assertThat(
+                lucenePersistedState.getLastAcceptedState().blocks().hasGlobalBlock(Metadata.CLUSTER_READ_ONLY_BLOCK),
+                equalTo(true)
+            );
+            assertThat(
+                IndexMetadata.INDEX_READ_ONLY_SETTING.get(
+                    lucenePersistedState.getLastAcceptedState().metadata().index("test-index1").getSettings()
+                ),
+                equalTo(true)
+            );
+        } finally {
+            IOUtils.close(gateway);
+        }
+    }
+
     private MockGatewayMetaState newGatewayForRemoteState(
         RemoteClusterStateService remoteClusterStateService,
         RemoteStoreRestoreService remoteStoreRestoreService,
         PersistedStateRegistry persistedStateRegistry,
-        ClusterState currentState
+        ClusterState currentState,
+        boolean prepareFullState
     ) throws IOException {
-        MockGatewayMetaState gateway = new MockGatewayMetaState(localNode, bigArrays);
+        MockGatewayMetaState gateway = new MockGatewayMetaState(localNode, bigArrays, prepareFullState);
         String randomRepoName = "randomRepoName";
         String stateRepoTypeAttributeKey = String.format(
             Locale.getDefault(),
@@ -963,6 +1035,7 @@ private MockGatewayMetaState newGatewayForRemoteState(
         when(clusterService.getClusterSettings()).thenReturn(
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
         );
+        when(transportService.getLocalNode()).thenReturn(mock(DiscoveryNode.class));
         final PersistedClusterStateService persistedClusterStateService = new PersistedClusterStateService(
             nodeEnvironment,
             xContentRegistry(),
diff --git a/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java b/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java
index d77596cf5cdd1..2f006a5519d69 100644
--- a/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java
+++ b/test/framework/src/main/java/org/opensearch/gateway/MockGatewayMetaState.java
@@ -67,6 +67,12 @@ public class MockGatewayMetaState extends GatewayMetaState {
     private final BigArrays bigArrays;
     private final RemoteClusterStateService remoteClusterStateService;
     private final RemoteStoreRestoreService remoteStoreRestoreService;
+    private boolean prepareFullState = false;
+
+    public MockGatewayMetaState(DiscoveryNode localNode, BigArrays bigArrays, boolean prepareFullState) {
+        this(localNode, bigArrays);
+        this.prepareFullState = prepareFullState;
+    }
 
     public MockGatewayMetaState(DiscoveryNode localNode, BigArrays bigArrays) {
         this.localNode = localNode;
@@ -99,8 +105,12 @@ Metadata upgradeMetadataForNode(
 
     @Override
     ClusterState prepareInitialClusterState(TransportService transportService, ClusterService clusterService, ClusterState clusterState) {
-        // Just set localNode here, not to mess with ClusterService and IndicesService mocking
-        return ClusterStateUpdaters.setLocalNode(clusterState, localNode);
+        if (prepareFullState) {
+            return super.prepareInitialClusterState(transportService, clusterService, clusterState);
+        } else {
+            // Just set localNode here, not to mess with ClusterService and IndicesService mocking
+            return ClusterStateUpdaters.setLocalNode(clusterState, localNode);
+        }
     }
 
     @Override
@@ -113,6 +123,16 @@ public void start(
         NodeEnvironment nodeEnvironment,
         NamedXContentRegistry xContentRegistry,
         PersistedStateRegistry persistedStateRegistry
+    ) {
+        start(settings, nodeEnvironment, xContentRegistry, persistedStateRegistry, false);
+    }
+
+    public void start(
+        Settings settings,
+        NodeEnvironment nodeEnvironment,
+        NamedXContentRegistry xContentRegistry,
+        PersistedStateRegistry persistedStateRegistry,
+        boolean prepareFullState
     ) {
         final TransportService transportService = mock(TransportService.class);
         when(transportService.getThreadPool()).thenReturn(mock(ThreadPool.class));
@@ -126,6 +146,7 @@ public void start(
         } catch (IOException e) {
             throw new AssertionError(e);
         }
+        this.prepareFullState = prepareFullState;
         start(
             settings,
             transportService,

From 64539d8ad7fe299e867ffb41b30513f8cbe50378 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Wed, 18 Oct 2023 23:23:10 +0530
Subject: [PATCH 184/388] fix change log conflicts (#10713)

Signed-off-by: bansvaru 
---
 CHANGELOG.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f4332c7847db6..0ad18b94f31b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,11 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
 - [Remote cluster state] Change file names for remote cluster state ([#10557](https://github.com/opensearch-project/OpenSearch/pull/10557))
 - [Remote cluster state] Upload global metadata in cluster state to remote store([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
-<<<<<<< Updated upstream
 - [Remote cluster state] Download functionality of global metadata from remote store ([#10535](https://github.com/opensearch-project/OpenSearch/pull/10535))
-=======
 - [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
->>>>>>> Stashed changes
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0

From 267bd5a84dfc62b265c053e5cd3abf31b9e01ff5 Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Wed, 18 Oct 2023 23:45:36 +0530
Subject: [PATCH 185/388] Version fixes for Resource Usage Stats and FS Stats
 (#10712)

Signed-off-by: Gaurav Bafna 
---
 .../action/admin/cluster/node/stats/NodeStats.java        | 4 ++--
 .../src/main/java/org/opensearch/monitor/fs/FsInfo.java   | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index 6ce6ca40cbce4..0c8aa027e5f01 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -206,7 +206,7 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             searchPipelineStats = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) { // make it 2.12 when we backport
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             resourceUsageStats = in.readOptionalWriteable(NodesResourceUsageStats::new);
         } else {
             resourceUsageStats = null;
@@ -462,7 +462,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_9_0)) {
             out.writeOptionalWriteable(searchPipelineStats);
         }
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) { // make it 2.12 when we backport
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(resourceUsageStats);
         }
         if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
index 4e2e9f280d765..8446ab0dd6166 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
@@ -335,7 +335,7 @@ public DeviceStats(StreamInput in) throws IOException {
             previousSectorsRead = in.readLong();
             currentSectorsWritten = in.readLong();
             previousSectorsWritten = in.readLong();
-            if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
                 currentReadTime = in.readLong();
                 previousReadTime = in.readLong();
                 currentWriteTime = in.readLong();
@@ -369,7 +369,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(previousSectorsRead);
             out.writeLong(currentSectorsWritten);
             out.writeLong(previousSectorsWritten);
-            if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
                 out.writeLong(currentReadTime);
                 out.writeLong(previousReadTime);
                 out.writeLong(currentWriteTime);
@@ -533,7 +533,7 @@ public IoStats(StreamInput in) throws IOException {
             this.totalWriteOperations = in.readLong();
             this.totalReadKilobytes = in.readLong();
             this.totalWriteKilobytes = in.readLong();
-            if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
                 this.totalReadTime = in.readLong();
                 this.totalWriteTime = in.readLong();
                 this.totalQueueSize = in.readLong();
@@ -557,7 +557,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(totalWriteOperations);
             out.writeLong(totalReadKilobytes);
             out.writeLong(totalWriteKilobytes);
-            if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
                 out.writeLong(totalReadTime);
                 out.writeLong(totalWriteTime);
                 out.writeLong(totalQueueSize);

From 5ec2fe9fb1b215be0da604f4f7e91a75d15f5f87 Mon Sep 17 00:00:00 2001
From: Kunal Kotwani 
Date: Wed, 18 Oct 2023 12:09:01 -0700
Subject: [PATCH 186/388] Add missing entry for 2.11 release notes (#10679)

Signed-off-by: Kunal Kotwani 
---
 release-notes/opensearch.release-notes-2.11.0.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release-notes/opensearch.release-notes-2.11.0.md b/release-notes/opensearch.release-notes-2.11.0.md
index 7ebf1b433c7c6..040cc053469ed 100644
--- a/release-notes/opensearch.release-notes-2.11.0.md
+++ b/release-notes/opensearch.release-notes-2.11.0.md
@@ -5,6 +5,7 @@
 ### Added
 - Add coordinator level stats for search latency ([#8386](https://github.com/opensearch-project/OpenSearch/issues/8386))
 - Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681))
+- Add parallel file download support for remote store based replication ([#8596](https://github.com/opensearch-project/OpenSearch/pull/8596)) 
 - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694))
 - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666))
 - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))

From a3c1d505903361a757945400c440aeac9e7ad973 Mon Sep 17 00:00:00 2001
From: Michael Froh 
Date: Wed, 18 Oct 2023 18:53:01 -0700
Subject: [PATCH 187/388] Add @abbashus and @adnapibar back to CODEOWNERS
 (#10681)

---
 .github/CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 4fa118e8486f1..8076adcf00ca9 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1 +1 @@
-*   @reta @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @peternied @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale @sohami @msfroh
+*   @abbashus @adnapibar @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kartg @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @ryanbogan @sachinpkale @saratvemulapalli @setiah @shwetathareja @sohami @tlfeng @VachaShah

From 7936f94ef6692516c58f4c46a8a3747da9cd9f1d Mon Sep 17 00:00:00 2001
From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com>
Date: Thu, 19 Oct 2023 08:40:13 +0530
Subject: [PATCH 188/388] Added close on buffered stream in s3 async upload for
 additional cleanup (#10710)

Signed-off-by: vikasvb90 
---
 .../s3/async/AsyncPartsHandler.java           | 44 +++++++++++-------
 .../s3/async/AsyncTransferManager.java        | 20 +++++---
 .../s3/async/AsyncTransferManagerTests.java   | 46 ++++++++++++++-----
 3 files changed, 75 insertions(+), 35 deletions(-)

diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
index 86bb70e5a40a2..6007d9f9c8a1c 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
@@ -29,6 +29,7 @@
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
@@ -140,28 +141,39 @@ private static void uploadPart(
         ExecutorService streamReadExecutor = uploadRequest.getWritePriority() == WritePriority.HIGH
             ? priorityExecutorService
             : executorService;
+        // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
+        // data can be retried instead of retrying whole file by the application.
+        InputStream inputStream = new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1));
         CompletableFuture uploadPartResponseFuture = SocketAccess.doPrivileged(
             () -> s3AsyncClient.uploadPart(
                 uploadPartRequest,
-                AsyncRequestBody.fromInputStream(
-                    // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
-                    // data can be retried instead of retrying whole file by the application.
-                    new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1)),
-                    inputStreamContainer.getContentLength(),
-                    streamReadExecutor
-                )
+                AsyncRequestBody.fromInputStream(inputStream, inputStreamContainer.getContentLength(), streamReadExecutor)
             )
         );
 
-        CompletableFuture convertFuture = uploadPartResponseFuture.thenApply(
-            uploadPartResponse -> convertUploadPartResponse(
-                completedParts,
-                inputStreamContainers,
-                uploadPartResponse,
-                partNumber,
-                uploadRequest.doRemoteDataIntegrityCheck()
-            )
-        );
+        CompletableFuture convertFuture = uploadPartResponseFuture.whenComplete((resp, throwable) -> {
+            try {
+                inputStream.close();
+            } catch (IOException ex) {
+                log.error(
+                    () -> new ParameterizedMessage(
+                        "Failed to close stream while uploading a part of idx {} and file {}.",
+                        uploadPartRequest.partNumber(),
+                        uploadPartRequest.key()
+                    ),
+                    ex
+                );
+            }
+        })
+            .thenApply(
+                uploadPartResponse -> convertUploadPartResponse(
+                    completedParts,
+                    inputStreamContainers,
+                    uploadPartResponse,
+                    partNumber,
+                    uploadRequest.doRemoteDataIntegrityCheck()
+                )
+            );
         futures.add(convertFuture);
 
         CompletableFutureUtils.forwardExceptionTo(convertFuture, uploadPartResponseFuture);
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
index db04636b89d50..a52745e33073e 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
@@ -40,6 +40,7 @@
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.List;
@@ -310,17 +311,22 @@ private void uploadInOneChunk(
         ExecutorService streamReadExecutor = uploadRequest.getWritePriority() == WritePriority.HIGH
             ? priorityExecutorService
             : executorService;
+        // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
+        // data can be retried instead of retrying whole file by the application.
+        InputStream inputStream = new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1));
         CompletableFuture putObjectFuture = SocketAccess.doPrivileged(
             () -> s3AsyncClient.putObject(
                 putObjectRequestBuilder.build(),
-                AsyncRequestBody.fromInputStream(
-                    // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
-                    // data can be retried instead of retrying whole file by the application.
-                    new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1)),
-                    inputStreamContainer.getContentLength(),
-                    streamReadExecutor
-                )
+                AsyncRequestBody.fromInputStream(inputStream, inputStreamContainer.getContentLength(), streamReadExecutor)
             ).handle((resp, throwable) -> {
+                try {
+                    inputStream.close();
+                } catch (IOException e) {
+                    log.error(
+                        () -> new ParameterizedMessage("Failed to close stream while uploading single file {}.", uploadRequest.getKey()),
+                        e
+                    );
+                }
                 if (throwable != null) {
                     Throwable unwrappedThrowable = ExceptionsHelper.unwrap(throwable, S3Exception.class);
                     if (unwrappedThrowable != null) {
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
index 607453986ab16..97a746cdeed93 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
@@ -37,9 +37,14 @@
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
@@ -71,17 +76,16 @@ public void testOneChunkUpload() {
             putObjectResponseCompletableFuture
         );
 
+        AtomicReference streamRef = new AtomicReference<>();
         CompletableFuture resultFuture = asyncTransferManager.uploadObject(
             s3AsyncClient,
             new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(1), WritePriority.HIGH, uploadSuccess -> {
                 // do nothing
             }, false, null),
-            new StreamContext(
-                (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position),
-                ByteSizeUnit.MB.toBytes(1),
-                ByteSizeUnit.MB.toBytes(1),
-                1
-            ),
+            new StreamContext((partIdx, partSize, position) -> {
+                streamRef.set(new ZeroInputStream(partSize));
+                return new InputStreamContainer(streamRef.get(), partSize, position);
+            }, ByteSizeUnit.MB.toBytes(1), ByteSizeUnit.MB.toBytes(1), 1),
             new StatsMetricPublisher()
         );
 
@@ -92,6 +96,14 @@ public void testOneChunkUpload() {
         }
 
         verify(s3AsyncClient, times(1)).putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class));
+
+        boolean closeError = false;
+        try {
+            streamRef.get().available();
+        } catch (IOException e) {
+            closeError = e.getMessage().equals("Stream closed");
+        }
+        assertTrue("InputStream was still open after upload", closeError);
     }
 
     public void testOneChunkUploadCorruption() {
@@ -162,17 +174,17 @@ public void testMultipartUpload() {
             abortMultipartUploadResponseCompletableFuture
         );
 
+        List streams = new ArrayList<>();
         CompletableFuture resultFuture = asyncTransferManager.uploadObject(
             s3AsyncClient,
             new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(5), WritePriority.HIGH, uploadSuccess -> {
                 // do nothing
             }, true, 3376132981L),
-            new StreamContext(
-                (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position),
-                ByteSizeUnit.MB.toBytes(1),
-                ByteSizeUnit.MB.toBytes(1),
-                5
-            ),
+            new StreamContext((partIdx, partSize, position) -> {
+                InputStream stream = new ZeroInputStream(partSize);
+                streams.add(stream);
+                return new InputStreamContainer(stream, partSize, position);
+            }, ByteSizeUnit.MB.toBytes(1), ByteSizeUnit.MB.toBytes(1), 5),
             new StatsMetricPublisher()
         );
 
@@ -182,6 +194,16 @@ public void testMultipartUpload() {
             fail("did not expect resultFuture to fail");
         }
 
+        streams.forEach(stream -> {
+            boolean closeError = false;
+            try {
+                stream.available();
+            } catch (IOException e) {
+                closeError = e.getMessage().equals("Stream closed");
+            }
+            assertTrue("InputStream was still open after upload", closeError);
+        });
+
         verify(s3AsyncClient, times(1)).createMultipartUpload(any(CreateMultipartUploadRequest.class));
         verify(s3AsyncClient, times(5)).uploadPart(any(UploadPartRequest.class), any(AsyncRequestBody.class));
         verify(s3AsyncClient, times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));

From 1d23b88cbf2177861f3a98ab906accf39c32c766 Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Thu, 19 Oct 2023 12:28:09 +0530
Subject: [PATCH 189/388] Muting s3 request stats test (#10736)

Signed-off-by: Gaurav Bafna 
---
 .../opensearch/repositories/s3/S3BlobStoreRepositoryTests.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index 1361f3165b653..4df30bfd2169e 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -165,6 +165,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
         return builder.build();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10735")
     @Override
     public void testRequestStats() throws Exception {
         final String repository = createRepository(randomName());

From 3899d117722e5517c3c02709cffc981bdf677fd5 Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Thu, 19 Oct 2023 17:49:27 +0530
Subject: [PATCH 190/388] Changing version for repo stats blob post backport to
 2.x (#10717)

Signed-off-by: Gaurav Bafna 
---
 .../opensearch/action/admin/cluster/node/stats/NodeStats.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index 0c8aa027e5f01..e9bfa358103c8 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -211,7 +211,7 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             resourceUsageStats = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             repositoriesStats = in.readOptionalWriteable(RepositoriesStats::new);
         } else {
             repositoriesStats = null;
@@ -465,7 +465,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(resourceUsageStats);
         }
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(repositoriesStats);
         }
     }

From 69f6f4e30909e215b4ca7fd55fd80cf8c4e8d3a4 Mon Sep 17 00:00:00 2001
From: "opensearch-trigger-bot[bot]"
 <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com>
Date: Thu, 19 Oct 2023 10:36:25 -0400
Subject: [PATCH 191/388] [AUTO] [main] Add bwc version 2.11.1. (#10648)

* Add bwc version 2.11.1

Signed-off-by: GitHub 

* Update Version.java

Signed-off-by: Andriy Redko 

---------

Signed-off-by: GitHub 
Signed-off-by: Andriy Redko 
Co-authored-by: opensearch-ci-bot 
Co-authored-by: Andriy Redko 
---
 .ci/bwcVersions                                     | 1 +
 libs/core/src/main/java/org/opensearch/Version.java | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index cfaadc5ed1e5e..144a8b71fca39 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -25,4 +25,5 @@ BWC_VERSION:
   - "2.10.0"
   - "2.10.1"
   - "2.11.0"
+  - "2.11.1"
   - "2.12.0"
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index eef4da719994c..8d9ee73a02c1d 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -96,6 +96,7 @@ public class Version implements Comparable, ToXContentFragment {
     public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_10_1 = new Version(2100199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0);
+    public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0);
     public static final Version CURRENT = V_3_0_0;

From da24ca756a3140f062e2c54d8fd0be88dc62e355 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Thu, 19 Oct 2023 15:25:46 -0400
Subject: [PATCH 192/388] Performance Improvement for Datetime formats (update
 version checks to 2.12.0) (#10754)

Signed-off-by: Andriy Redko 
---
 .../src/main/java/org/opensearch/search/DocValueFormat.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/search/DocValueFormat.java b/server/src/main/java/org/opensearch/search/DocValueFormat.java
index 412191c57abd8..7be51643eeb7d 100644
--- a/server/src/main/java/org/opensearch/search/DocValueFormat.java
+++ b/server/src/main/java/org/opensearch/search/DocValueFormat.java
@@ -243,7 +243,7 @@ public DateTime(DateFormatter formatter, ZoneId timeZone, DateFieldMapper.Resolu
         }
 
         public DateTime(StreamInput in) throws IOException {
-            if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
                 this.formatter = DateFormatter.forPattern(in.readString(), in.readOptionalString());
             } else {
                 this.formatter = DateFormatter.forPattern(in.readString());
@@ -265,12 +265,12 @@ public String getWriteableName() {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().before(Version.V_3_0_0) && formatter.equals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER)) {
+            if (out.getVersion().before(Version.V_2_12_0) && formatter.equals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER)) {
                 out.writeString(DateFieldMapper.LEGACY_DEFAULT_DATE_TIME_FORMATTER.pattern()); // required for backwards compatibility
             } else {
                 out.writeString(formatter.pattern());
             }
-            if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
                 out.writeOptionalString(formatter.printPattern());
             }
             out.writeString(timeZone.getId());

From e389a09640cf4d687ae5bbe59d36f5e15624e985 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Thu, 19 Oct 2023 13:31:01 -0700
Subject: [PATCH 193/388] Fix bug where retries within
 RemoteStoreRefreshListener cause infos/checkpoint mismatch (#10655)

* Fix bug where retries within RemoteStoreRefreshListener cause mismatch between ReplicationCheckpoint and uploaded SegmentInfos.

Retries within RemoteStoreRefreshListener run outside of the refresh thread.  This means that concurrent refreshes
may occur during syncSegments execution updating the on-reader SegmentInfos.  A shard's latest ReplicationCheckpoint
is computed and set in a refresh listener, but it is not guaranteed the listener has run before the retry fetches the infos or checkpoint independently.
This fix ensures the listener recomputes the checkpoint while fetching the SegmentInfos. This change also
ensures that we only recompute the checkpoint when necessary because it comes with an IO cost to compute StoreFileMetadata.

Signed-off-by: Marc Handalian 

Update refresh listener to recompute checkpoint from latest infos snapshot.

Signed-off-by: Marc Handalian 

Fix broken test case by comparing segments gen

Signed-off-by: Marc Handalian 

spotless

Signed-off-by: Marc Handalian 

Fix RemoteStoreRefreshListener tests

Signed-off-by: Marc Handalian 

* add extra log

Signed-off-by: Marc Handalian 

---------

Signed-off-by: Marc Handalian 
---
 .../opensearch/index/shard/IndexShard.java    | 67 +++++++++++--------
 .../shard/RemoteStoreRefreshListener.java     |  6 +-
 .../RemoteStoreRefreshListenerTests.java      |  4 +-
 .../SegmentReplicationIndexShardTests.java    | 27 ++++++++
 4 files changed, 70 insertions(+), 34 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 9489c7d7fc1dd..5ebfd3863a6cf 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -1608,8 +1608,11 @@ public GatedCloseable acquireSafeIndexCommit() throws EngineExcepti
     }
 
     /**
-     * Compute and return the latest ReplicationCheckpoint for a particular shard.
-     * @return EMPTY checkpoint before the engine is opened and null for non-segrep enabled indices
+     * return the most recently computed ReplicationCheckpoint for a particular shard.
+     * The checkpoint is updated inside a refresh listener and may lag behind the SegmentInfos on the reader.
+     * To guarantee the checkpoint is upto date with the latest on-reader infos, use `getLatestSegmentInfosAndCheckpoint` instead.
+     *
+     * @return {@link ReplicationCheckpoint} - The most recently computed ReplicationCheckpoint.
      */
     public ReplicationCheckpoint getLatestReplicationCheckpoint() {
         return replicationTracker.getLatestReplicationCheckpoint();
@@ -1628,34 +1631,12 @@ public ReplicationCheckpoint getLatestReplicationCheckpoint() {
     public Tuple, ReplicationCheckpoint> getLatestSegmentInfosAndCheckpoint() {
         assert indexSettings.isSegRepEnabled();
 
-        Tuple, ReplicationCheckpoint> nullSegmentInfosEmptyCheckpoint = new Tuple<>(
-            new GatedCloseable<>(null, () -> {}),
-            getLatestReplicationCheckpoint()
-        );
-
-        if (getEngineOrNull() == null) {
-            return nullSegmentInfosEmptyCheckpoint;
-        }
         // do not close the snapshot - caller will close it.
         GatedCloseable snapshot = null;
         try {
             snapshot = getSegmentInfosSnapshot();
-            if (snapshot.get() != null) {
-                SegmentInfos segmentInfos = snapshot.get();
-                final Map metadataMap = store.getSegmentMetadataMap(segmentInfos);
-                return new Tuple<>(
-                    snapshot,
-                    new ReplicationCheckpoint(
-                        this.shardId,
-                        getOperationPrimaryTerm(),
-                        segmentInfos.getGeneration(),
-                        segmentInfos.getVersion(),
-                        metadataMap.values().stream().mapToLong(StoreFileMetadata::length).sum(),
-                        getEngine().config().getCodec().getName(),
-                        metadataMap
-                    )
-                );
-            }
+            final SegmentInfos segmentInfos = snapshot.get();
+            return new Tuple<>(snapshot, computeReplicationCheckpoint(segmentInfos));
         } catch (IOException | AlreadyClosedException e) {
             logger.error("Error Fetching SegmentInfos and latest checkpoint", e);
             if (snapshot != null) {
@@ -1666,7 +1647,39 @@ public Tuple, ReplicationCheckpoint> getLatestSegme
                 }
             }
         }
-        return nullSegmentInfosEmptyCheckpoint;
+        return new Tuple<>(new GatedCloseable<>(null, () -> {}), getLatestReplicationCheckpoint());
+    }
+
+    /**
+     * Compute the latest {@link ReplicationCheckpoint} from a SegmentInfos.
+     * This function fetches a metadata snapshot from the store that comes with an IO cost.
+     * We will reuse the existing stored checkpoint if it is at the same SI version.
+     *
+     * @param segmentInfos {@link SegmentInfos} infos to use to compute.
+     * @return {@link ReplicationCheckpoint} Checkpoint computed from the infos.
+     * @throws IOException When there is an error computing segment metadata from the store.
+     */
+    ReplicationCheckpoint computeReplicationCheckpoint(SegmentInfos segmentInfos) throws IOException {
+        if (segmentInfos == null) {
+            return ReplicationCheckpoint.empty(shardId);
+        }
+        final ReplicationCheckpoint latestReplicationCheckpoint = getLatestReplicationCheckpoint();
+        if (latestReplicationCheckpoint.getSegmentInfosVersion() == segmentInfos.getVersion()
+            && latestReplicationCheckpoint.getSegmentsGen() == segmentInfos.getGeneration()) {
+            return latestReplicationCheckpoint;
+        }
+        final Map metadataMap = store.getSegmentMetadataMap(segmentInfos);
+        final ReplicationCheckpoint checkpoint = new ReplicationCheckpoint(
+            this.shardId,
+            getOperationPrimaryTerm(),
+            segmentInfos.getGeneration(),
+            segmentInfos.getVersion(),
+            metadataMap.values().stream().mapToLong(StoreFileMetadata::length).sum(),
+            getEngine().config().getCodec().getName(),
+            metadataMap
+        );
+        logger.trace("Recomputed ReplicationCheckpoint for shard {}", checkpoint);
+        return checkpoint;
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index 698e61f6f7a09..c650edc31da8d 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -181,7 +181,6 @@ private boolean syncSegments() {
             // in the remote store.
             return indexShard.state() != IndexShardState.STARTED || !(indexShard.getEngine() instanceof InternalEngine);
         }
-        ReplicationCheckpoint checkpoint = indexShard.getLatestReplicationCheckpoint();
         beforeSegmentsSync();
         long refreshTimeMs = segmentTracker.getLocalRefreshTimeMs(), refreshClockTimeMs = segmentTracker.getLocalRefreshClockTimeMs();
         long refreshSeqNo = segmentTracker.getLocalRefreshSeqNo();
@@ -199,10 +198,7 @@ private boolean syncSegments() {
 
                 try (GatedCloseable segmentInfosGatedCloseable = indexShard.getSegmentInfosSnapshot()) {
                     SegmentInfos segmentInfos = segmentInfosGatedCloseable.get();
-                    assert segmentInfos.getGeneration() == checkpoint.getSegmentsGen() : "SegmentInfos generation: "
-                        + segmentInfos.getGeneration()
-                        + " does not match metadata generation: "
-                        + checkpoint.getSegmentsGen();
+                    final ReplicationCheckpoint checkpoint = indexShard.computeReplicationCheckpoint(segmentInfos);
                     // Capture replication checkpoint before uploading the segments as upload can take some time and checkpoint can
                     // move.
                     long lastRefreshedCheckpoint = ((InternalEngine) indexShard.getEngine()).lastRefreshedCheckpoint();
diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
index 5a13f57db2c87..51814283c5eb3 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
@@ -520,8 +520,8 @@ private Tuple mockIn
             if (counter.incrementAndGet() <= succeedOnAttempt) {
                 throw new RuntimeException("Inducing failure in upload");
             }
-            return indexShard.getLatestSegmentInfosAndCheckpoint();
-        })).when(shard).getLatestSegmentInfosAndCheckpoint();
+            return indexShard.getLatestReplicationCheckpoint();
+        })).when(shard).computeReplicationCheckpoint(any());
 
         doAnswer(invocation -> {
             if (Objects.nonNull(successLatch)) {
diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
index 52f28aead533d..eab38bfe5c64d 100644
--- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
@@ -925,6 +925,33 @@ public void testSnapshotWhileFailoverIncomplete() throws Exception {
         }
     }
 
+    public void testReuseReplicationCheckpointWhenLatestInfosIsUnChanged() throws Exception {
+        try (ReplicationGroup shards = createGroup(1, settings, indexMapping, new NRTReplicationEngineFactory(), createTempDir())) {
+            final IndexShard primaryShard = shards.getPrimary();
+            shards.startAll();
+            shards.indexDocs(10);
+            shards.refresh("test");
+            replicateSegments(primaryShard, shards.getReplicas());
+            shards.assertAllEqual(10);
+            final ReplicationCheckpoint latestReplicationCheckpoint = primaryShard.getLatestReplicationCheckpoint();
+            try (GatedCloseable segmentInfosSnapshot = primaryShard.getSegmentInfosSnapshot()) {
+                assertEquals(latestReplicationCheckpoint, primaryShard.computeReplicationCheckpoint(segmentInfosSnapshot.get()));
+            }
+            final Tuple, ReplicationCheckpoint> latestSegmentInfosAndCheckpoint = primaryShard
+                .getLatestSegmentInfosAndCheckpoint();
+            try (final GatedCloseable closeable = latestSegmentInfosAndCheckpoint.v1()) {
+                assertEquals(latestReplicationCheckpoint, primaryShard.computeReplicationCheckpoint(closeable.get()));
+            }
+        }
+    }
+
+    public void testComputeReplicationCheckpointNullInfosReturnsEmptyCheckpoint() throws Exception {
+        try (ReplicationGroup shards = createGroup(1, settings, indexMapping, new NRTReplicationEngineFactory(), createTempDir())) {
+            final IndexShard primaryShard = shards.getPrimary();
+            assertEquals(ReplicationCheckpoint.empty(primaryShard.shardId), primaryShard.computeReplicationCheckpoint(null));
+        }
+    }
+
     private SnapshotShardsService getSnapshotShardsService(IndexShard replicaShard) {
         final TransportService transportService = mock(TransportService.class);
         when(transportService.getThreadPool()).thenReturn(threadPool);

From 781968b2e53f3214b73d4d8e7c1baa572b334f27 Mon Sep 17 00:00:00 2001
From: Siddhant Deshmukh 
Date: Thu, 19 Oct 2023 14:39:29 -0700
Subject: [PATCH 194/388] Categorize search queries by type and log query shape
 (#10724)

* Search Query Categorizor initial skeleton using QueryBuilderVisitor

Signed-off-by: Siddhant Deshmukh 

* Integrate metrics framework, add counters and log query shape

Signed-off-by: Siddhant Deshmukh 

* Update changelog

Signed-off-by: Siddhant Deshmukh 

* Add level attribute to QueryBuilderVisitor and as a tag in Counters

Signed-off-by: Siddhant Deshmukh 

* Log query shape as debug log

Signed-off-by: Siddhant Deshmukh 

* Integrate metrics framework, refactor code and update tests

Signed-off-by: Siddhant Deshmukh 

* Fix build

Signed-off-by: Siddhant Deshmukh 

* Add javadocs

Signed-off-by: Siddhant Deshmukh 

* Minor fix

Signed-off-by: Siddhant Deshmukh 

* Spotless check changes

Signed-off-by: Siddhant Deshmukh 

* Address comments, add agg and sort counters, add feature flag, refactoring

Signed-off-by: Siddhant Deshmukh 

* Build fix

Signed-off-by: Siddhant Deshmukh 

* spotless check

Signed-off-by: Siddhant Deshmukh 

* Fix tests

Signed-off-by: Siddhant Deshmukh 

* Dynamic feature flag with callback

Signed-off-by: Siddhant Deshmukh 

* Minor fix

Signed-off-by: Siddhant Deshmukh 

* Add initialization in callback

Signed-off-by: Siddhant Deshmukh 

* Address comments

Signed-off-by: Siddhant Deshmukh 

* Add exception handling

Signed-off-by: Siddhant Deshmukh 

* Refactoring and renaming

Signed-off-by: Siddhant Deshmukh 

* Minor fix

Signed-off-by: Siddhant Deshmukh 

* Fix changelog and minor refactoring

Signed-off-by: Siddhant Deshmukh 

* Address review comments

Signed-off-by: Siddhant Deshmukh 

* Add unit tests

Signed-off-by: Siddhant Deshmukh 

* Address review comments and add complex query unit test

Signed-off-by: Siddhant Deshmukh 

* Add sort order as a tag to sort counter

Signed-off-by: Siddhant Deshmukh 

* Address review comments

Signed-off-by: Siddhant Deshmukh 

* Address final comments

Signed-off-by: Siddhant Deshmukh 

* Build fix

Signed-off-by: Siddhant Deshmukh 

* Fix build tests failure

Signed-off-by: Siddhant Deshmukh 

* Minor fix

Signed-off-by: Siddhant Deshmukh 

* Minor fix

Signed-off-by: Siddhant Deshmukh 

* Empty commit

Signed-off-by: Siddhant Deshmukh 

* Remove extra newline

Signed-off-by: Michael Froh 

* Empty commit

Signed-off-by: Siddhant Deshmukh 

---------

Signed-off-by: Siddhant Deshmukh 
Signed-off-by: Michael Froh 
Co-authored-by: Michael Froh 
---
 CHANGELOG.md                                  |   1 +
 .../action/search/SearchQueryCategorizer.java |  81 +++++++
 .../SearchQueryCategorizingVisitor.java       |  73 ++++++
 .../action/search/SearchQueryCounters.java    | 117 +++++++++
 .../action/search/TransportSearchAction.java  |  36 ++-
 .../common/settings/ClusterSettings.java      |   1 +
 .../index/query/QueryShapeVisitor.java        |  86 +++++++
 .../search/SearchQueryCategorizerTests.java   | 228 ++++++++++++++++++
 .../index/query/QueryShapeVisitorTests.java   |  31 +++
 .../snapshots/SnapshotResiliencyTests.java    |   4 +-
 10 files changed, 656 insertions(+), 2 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
 create mode 100644 server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java
 create mode 100644 server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java
 create mode 100644 server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java
 create mode 100644 server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ad18b94f31b7..552c277789dd7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -91,6 +91,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
 - [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567))
+- Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
new file mode 100644
index 0000000000000..9cbe2d2ffcb7d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.search;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryBuilderVisitor;
+import org.opensearch.index.query.QueryShapeVisitor;
+import org.opensearch.search.aggregations.AggregatorFactories;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.sort.SortBuilder;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.tags.Tags;
+
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * Class to categorize the search queries based on the type and increment the relevant counters.
+ * Class also logs the query shape.
+ */
+final class SearchQueryCategorizer {
+
+    private static final Logger log = LogManager.getLogger(SearchQueryCategorizer.class);
+
+    final SearchQueryCounters searchQueryCounters;
+
+    public SearchQueryCategorizer(MetricsRegistry metricsRegistry) {
+        searchQueryCounters = new SearchQueryCounters(metricsRegistry);
+    }
+
+    public void categorize(SearchSourceBuilder source) {
+        QueryBuilder topLevelQueryBuilder = source.query();
+
+        logQueryShape(topLevelQueryBuilder);
+        incrementQueryTypeCounters(topLevelQueryBuilder);
+        incrementQueryAggregationCounters(source.aggregations());
+        incrementQuerySortCounters(source.sorts());
+    }
+
+    private void incrementQuerySortCounters(List> sorts) {
+        if (sorts != null && sorts.size() > 0) {
+            for (ListIterator> it = sorts.listIterator(); it.hasNext();) {
+                SortBuilder sortBuilder = it.next();
+                String sortOrder = sortBuilder.order().toString();
+                searchQueryCounters.sortCounter.add(1, Tags.create().addTag("sort_order", sortOrder));
+            }
+        }
+    }
+
+    private void incrementQueryAggregationCounters(AggregatorFactories.Builder aggregations) {
+        if (aggregations != null) {
+            searchQueryCounters.aggCounter.add(1);
+        }
+    }
+
+    private void incrementQueryTypeCounters(QueryBuilder topLevelQueryBuilder) {
+        if (topLevelQueryBuilder == null) {
+            return;
+        }
+        QueryBuilderVisitor searchQueryVisitor = new SearchQueryCategorizingVisitor(searchQueryCounters);
+        topLevelQueryBuilder.visit(searchQueryVisitor);
+    }
+
+    private void logQueryShape(QueryBuilder topLevelQueryBuilder) {
+        if (topLevelQueryBuilder == null) {
+            return;
+        }
+        QueryShapeVisitor shapeVisitor = new QueryShapeVisitor();
+        topLevelQueryBuilder.visit(shapeVisitor);
+        log.debug("Query shape : {}", shapeVisitor.prettyPrintTree("  "));
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java
new file mode 100644
index 0000000000000..98f0169e69a5c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java
@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.search;
+
+import org.apache.lucene.search.BooleanClause;
+import org.opensearch.index.query.BoolQueryBuilder;
+import org.opensearch.index.query.MatchPhraseQueryBuilder;
+import org.opensearch.index.query.MatchQueryBuilder;
+import org.opensearch.index.query.MultiMatchQueryBuilder;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryBuilderVisitor;
+import org.opensearch.index.query.QueryStringQueryBuilder;
+import org.opensearch.index.query.RangeQueryBuilder;
+import org.opensearch.index.query.RegexpQueryBuilder;
+import org.opensearch.index.query.TermQueryBuilder;
+import org.opensearch.index.query.WildcardQueryBuilder;
+import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.opensearch.telemetry.metrics.tags.Tags;
+
+/**
+ * Class to visit the querybuilder tree and also track the level information.
+ * Increments the counters related to Search Query type.
+ */
+final class SearchQueryCategorizingVisitor implements QueryBuilderVisitor {
+    private static final String LEVEL_TAG = "level";
+    private final int level;
+    private final SearchQueryCounters searchQueryCounters;
+
+    public SearchQueryCategorizingVisitor(SearchQueryCounters searchQueryCounters) {
+        this(searchQueryCounters, 0);
+    }
+
+    private SearchQueryCategorizingVisitor(SearchQueryCounters counters, int level) {
+        this.searchQueryCounters = counters;
+        this.level = level;
+    }
+
+    public void accept(QueryBuilder qb) {
+        if (qb instanceof BoolQueryBuilder) {
+            searchQueryCounters.boolCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof FunctionScoreQueryBuilder) {
+            searchQueryCounters.functionScoreCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof MatchQueryBuilder) {
+            searchQueryCounters.matchCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof MatchPhraseQueryBuilder) {
+            searchQueryCounters.matchPhrasePrefixCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof MultiMatchQueryBuilder) {
+            searchQueryCounters.multiMatchCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof QueryStringQueryBuilder) {
+            searchQueryCounters.queryStringQueryCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof RangeQueryBuilder) {
+            searchQueryCounters.rangeCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof RegexpQueryBuilder) {
+            searchQueryCounters.regexCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof TermQueryBuilder) {
+            searchQueryCounters.termCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else if (qb instanceof WildcardQueryBuilder) {
+            searchQueryCounters.wildcardCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        } else {
+            searchQueryCounters.otherQueryCounter.add(1, Tags.create().addTag(LEVEL_TAG, level));
+        }
+    }
+
+    public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) {
+        return new SearchQueryCategorizingVisitor(searchQueryCounters, level + 1);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java b/server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java
new file mode 100644
index 0000000000000..7e0259af07701
--- /dev/null
+++ b/server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java
@@ -0,0 +1,117 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.search;
+
+import org.opensearch.telemetry.metrics.Counter;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+
+/**
+ * Class contains all the Counters related to search query types.
+ */
+final class SearchQueryCounters {
+    private static final String UNIT = "1";
+    private final MetricsRegistry metricsRegistry;
+
+    // Counters related to Query types
+    public final Counter aggCounter;
+    public final Counter boolCounter;
+    public final Counter functionScoreCounter;
+    public final Counter matchCounter;
+    public final Counter matchPhrasePrefixCounter;
+    public final Counter multiMatchCounter;
+    public final Counter otherQueryCounter;
+    public final Counter queryStringQueryCounter;
+    public final Counter rangeCounter;
+    public final Counter regexCounter;
+
+    public final Counter sortCounter;
+    public final Counter skippedCounter;
+    public final Counter termCounter;
+    public final Counter totalCounter;
+    public final Counter wildcardCounter;
+
+    public SearchQueryCounters(MetricsRegistry metricsRegistry) {
+        this.metricsRegistry = metricsRegistry;
+        this.aggCounter = metricsRegistry.createCounter(
+            "search.query.type.agg.count",
+            "Counter for the number of top level agg search queries",
+            UNIT
+        );
+        this.boolCounter = metricsRegistry.createCounter(
+            "search.query.type.bool.count",
+            "Counter for the number of top level and nested bool search queries",
+            UNIT
+        );
+        this.functionScoreCounter = metricsRegistry.createCounter(
+            "search.query.type.functionscore.count",
+            "Counter for the number of top level and nested function score search queries",
+            UNIT
+        );
+        this.matchCounter = metricsRegistry.createCounter(
+            "search.query.type.match.count",
+            "Counter for the number of top level and nested match search queries",
+            UNIT
+        );
+        this.matchPhrasePrefixCounter = metricsRegistry.createCounter(
+            "search.query.type.matchphrase.count",
+            "Counter for the number of top level and nested match phrase prefix search queries",
+            UNIT
+        );
+        this.multiMatchCounter = metricsRegistry.createCounter(
+            "search.query.type.multimatch.count",
+            "Counter for the number of top level and nested multi match search queries",
+            UNIT
+        );
+        this.otherQueryCounter = metricsRegistry.createCounter(
+            "search.query.type.other.count",
+            "Counter for the number of top level and nested search queries that do not match any other categories",
+            UNIT
+        );
+        this.queryStringQueryCounter = metricsRegistry.createCounter(
+            "search.query.type.querystringquery.count",
+            "Counter for the number of top level and nested queryStringQuery search queries",
+            UNIT
+        );
+        this.rangeCounter = metricsRegistry.createCounter(
+            "search.query.type.range.count",
+            "Counter for the number of top level and nested range search queries",
+            UNIT
+        );
+        this.regexCounter = metricsRegistry.createCounter(
+            "search.query.type.regex.count",
+            "Counter for the number of top level and nested regex search queries",
+            UNIT
+        );
+        this.skippedCounter = metricsRegistry.createCounter(
+            "search.query.type.skipped.count",
+            "Counter for the number queries skipped due to error",
+            UNIT
+        );
+        this.sortCounter = metricsRegistry.createCounter(
+            "search.query.type.sort.count",
+            "Counter for the number of top level sort search queries",
+            UNIT
+        );
+        this.termCounter = metricsRegistry.createCounter(
+            "search.query.type.term.count",
+            "Counter for the number of top level and nested term search queries",
+            UNIT
+        );
+        this.totalCounter = metricsRegistry.createCounter(
+            "search.query.type.total.count",
+            "Counter for the number of top level and nested search queries",
+            UNIT
+        );
+        this.wildcardCounter = metricsRegistry.createCounter(
+            "search.query.type.wildcard.count",
+            "Counter for the number of top level and nested wildcard search queries",
+            UNIT
+        );
+    }
+}
diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
index 284f71bd9da62..a6fb8453af4ff 100644
--- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
@@ -88,6 +88,7 @@
 import org.opensearch.search.profile.SearchProfileShardResults;
 import org.opensearch.tasks.CancellableTask;
 import org.opensearch.tasks.Task;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.RemoteClusterAware;
 import org.opensearch.transport.RemoteClusterService;
@@ -137,6 +138,13 @@ public class TransportSearchAction extends HandledTransportAction SEARCH_QUERY_METRICS_ENABLED_SETTING = Setting.boolSetting(
+        "search.query.metrics.enabled",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     // cluster level setting for timeout based search cancellation. If search request level parameter is present then that will take
     // precedence over the cluster setting value
     public static final String SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING_KEY = "search.cancel_after_time_interval";
@@ -177,8 +185,14 @@ public class TransportSearchAction extends HandledTransportAction) SearchRequest::new);
         this.client = client;
@@ -211,6 +226,17 @@ public TransportSearchAction(
         this.isRequestStatsEnabled = clusterService.getClusterSettings().get(SEARCH_REQUEST_STATS_ENABLED);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(SEARCH_REQUEST_STATS_ENABLED, this::setIsRequestStatsEnabled);
         this.searchRequestStats = searchRequestStats;
+        this.metricsRegistry = metricsRegistry;
+        this.searchQueryMetricsEnabled = clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING);
+        clusterService.getClusterSettings()
+            .addSettingsUpdateConsumer(SEARCH_QUERY_METRICS_ENABLED_SETTING, this::setSearchQueryMetricsEnabled);
+    }
+
+    private void setSearchQueryMetricsEnabled(boolean searchQueryMetricsEnabled) {
+        this.searchQueryMetricsEnabled = searchQueryMetricsEnabled;
+        if ((this.searchQueryMetricsEnabled == true) && this.searchQueryCategorizer == null) {
+            this.searchQueryCategorizer = new SearchQueryCategorizer(metricsRegistry);
+        }
     }
 
     private void setIsRequestStatsEnabled(boolean isRequestStatsEnabled) {
@@ -489,6 +515,14 @@ private void executeRequest(
             return;
         }
 
+        if (searchQueryMetricsEnabled) {
+            try {
+                searchQueryCategorizer.categorize(searchRequest.source());
+            } catch (Exception e) {
+                logger.error("Error while trying to categorize the query.", e);
+            }
+        }
+
         ActionListener rewriteListener = ActionListener.wrap(source -> {
             if (source != searchRequest.source()) {
                 // only set it if it changed - we don't allow null values to be set but it might be already null. this way we catch
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 90f91dcb7c553..76883c200542e 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -377,6 +377,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 TransportSearchAction.SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING,
                 TransportSearchAction.SEARCH_REQUEST_STATS_ENABLED,
                 TransportSearchAction.SEARCH_PHASE_TOOK_ENABLED,
+                TransportSearchAction.SEARCH_QUERY_METRICS_ENABLED_SETTING,
                 RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE,
                 SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER,
                 RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
diff --git a/server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java b/server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java
new file mode 100644
index 0000000000000..3ba13bc7a2da4
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/query/QueryShapeVisitor.java
@@ -0,0 +1,86 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.query;
+
+import org.apache.lucene.search.BooleanClause;
+import org.opensearch.common.SetOnce;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Class to traverse the QueryBuilder tree and capture the query shape
+ */
+public final class QueryShapeVisitor implements QueryBuilderVisitor {
+    private final SetOnce queryType = new SetOnce<>();
+    private final Map> childVisitors = new EnumMap<>(BooleanClause.Occur.class);
+
+    @Override
+    public void accept(QueryBuilder qb) {
+        queryType.set(qb.getName());
+    }
+
+    @Override
+    public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) {
+        // Should get called once per Occur value
+        if (childVisitors.containsKey(occur)) {
+            throw new IllegalStateException("child visitor already called for " + occur);
+        }
+        final List childVisitorList = new ArrayList<>();
+        QueryBuilderVisitor childVisitorWrapper = new QueryBuilderVisitor() {
+            QueryShapeVisitor currentChild;
+
+            @Override
+            public void accept(QueryBuilder qb) {
+                currentChild = new QueryShapeVisitor();
+                childVisitorList.add(currentChild);
+                currentChild.accept(qb);
+            }
+
+            @Override
+            public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) {
+                return currentChild.getChildVisitor(occur);
+            }
+        };
+        childVisitors.put(occur, childVisitorList);
+        return childVisitorWrapper;
+    }
+
+    String toJson() {
+        StringBuilder outputBuilder = new StringBuilder("{\"type\":\"").append(queryType.get()).append("\"");
+        for (Map.Entry> entry : childVisitors.entrySet()) {
+            outputBuilder.append(",\"").append(entry.getKey().name().toLowerCase(Locale.ROOT)).append("\"[");
+            boolean first = true;
+            for (QueryShapeVisitor child : entry.getValue()) {
+                if (!first) {
+                    outputBuilder.append(",");
+                }
+                outputBuilder.append(child.toJson());
+                first = false;
+            }
+            outputBuilder.append("]");
+        }
+        outputBuilder.append("}");
+        return outputBuilder.toString();
+    }
+
+    public String prettyPrintTree(String indent) {
+        StringBuilder outputBuilder = new StringBuilder(indent).append(queryType.get()).append("\n");
+        for (Map.Entry> entry : childVisitors.entrySet()) {
+            outputBuilder.append(indent).append("  ").append(entry.getKey().name().toLowerCase(Locale.ROOT)).append(":\n");
+            for (QueryShapeVisitor child : entry.getValue()) {
+                outputBuilder.append(child.prettyPrintTree(indent + "    "));
+            }
+        }
+        return outputBuilder.toString();
+    }
+}
diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java
new file mode 100644
index 0000000000000..a2e301143d694
--- /dev/null
+++ b/server/src/test/java/org/opensearch/action/search/SearchQueryCategorizerTests.java
@@ -0,0 +1,228 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.search;
+
+import org.opensearch.index.query.BoolQueryBuilder;
+import org.opensearch.index.query.BoostingQueryBuilder;
+import org.opensearch.index.query.MatchNoneQueryBuilder;
+import org.opensearch.index.query.MatchQueryBuilder;
+import org.opensearch.index.query.MultiMatchQueryBuilder;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.QueryStringQueryBuilder;
+import org.opensearch.index.query.RangeQueryBuilder;
+import org.opensearch.index.query.RegexpQueryBuilder;
+import org.opensearch.index.query.TermQueryBuilder;
+import org.opensearch.index.query.WildcardQueryBuilder;
+import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder;
+import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.sort.ScoreSortBuilder;
+import org.opensearch.search.sort.SortOrder;
+import org.opensearch.telemetry.metrics.Counter;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.tags.Tags;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
+
+import java.util.Arrays;
+
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+public final class SearchQueryCategorizerTests extends OpenSearchTestCase {
+
+    private MetricsRegistry metricsRegistry;
+
+    private SearchQueryCategorizer searchQueryCategorizer;
+
+    @Before
+    public void setup() {
+        metricsRegistry = mock(MetricsRegistry.class);
+        when(metricsRegistry.createCounter(any(String.class), any(String.class), any(String.class))).thenAnswer(
+            invocation -> mock(Counter.class)
+        );
+        searchQueryCategorizer = new SearchQueryCategorizer(metricsRegistry);
+    }
+
+    public void testAggregationsQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.aggregation(
+            new MultiTermsAggregationBuilder("agg1").terms(
+                Arrays.asList(
+                    new MultiTermsValuesSourceConfig.Builder().setFieldName("username").build(),
+                    new MultiTermsValuesSourceConfig.Builder().setFieldName("rating").build()
+                )
+            )
+        );
+        sourceBuilder.size(0);
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.aggCounter).add(eq(1.0d));
+    }
+
+    public void testBoolQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        sourceBuilder.query(new BoolQueryBuilder().must(new MatchQueryBuilder("searchText", "fox")));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.boolCounter).add(eq(1.0d), any(Tags.class));
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.matchCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testFunctionScoreQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        sourceBuilder.query(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro")));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.functionScoreCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testMatchQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        sourceBuilder.query(QueryBuilders.matchQuery("tags", "php"));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.matchCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testMatchPhraseQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        sourceBuilder.query(QueryBuilders.matchPhraseQuery("tags", "php"));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.matchPhrasePrefixCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testMultiMatchQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        sourceBuilder.query(new MultiMatchQueryBuilder("foo bar", "myField"));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.multiMatchCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testOtherQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        BoostingQueryBuilder queryBuilder = new BoostingQueryBuilder(
+            new TermQueryBuilder("unmapped_field", "foo"),
+            new MatchNoneQueryBuilder()
+        );
+        sourceBuilder.query(queryBuilder);
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.otherQueryCounter, times(2)).add(eq(1.0d), any(Tags.class));
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.termCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testQueryStringQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder("foo:*");
+        sourceBuilder.query(queryBuilder);
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.queryStringQueryCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testRangeQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        RangeQueryBuilder rangeQuery = new RangeQueryBuilder("date");
+        rangeQuery.gte("1970-01-01");
+        rangeQuery.lt("1982-01-01");
+        sourceBuilder.query(rangeQuery);
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.rangeCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testRegexQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.query(new RegexpQueryBuilder("field", "text"));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.regexCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testSortQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.query(QueryBuilders.matchQuery("tags", "ruby"));
+        sourceBuilder.sort("creationDate", SortOrder.DESC);
+        sourceBuilder.sort(new ScoreSortBuilder());
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.matchCounter).add(eq(1.0d), any(Tags.class));
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.sortCounter, times(2)).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testTermQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        sourceBuilder.query(QueryBuilders.termQuery("field", "value2"));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.termCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testWildcardQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+        sourceBuilder.query(new WildcardQueryBuilder("field", "text"));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.wildcardCounter).add(eq(1.0d), any(Tags.class));
+    }
+
+    public void testComplexQuery() {
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+        sourceBuilder.size(50);
+
+        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("field", "value2");
+        MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("tags", "php");
+        RegexpQueryBuilder regexpQueryBuilder = new RegexpQueryBuilder("field", "text");
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().must(termQueryBuilder)
+            .filter(matchQueryBuilder)
+            .should(regexpQueryBuilder);
+        sourceBuilder.query(boolQueryBuilder);
+        sourceBuilder.aggregation(new RangeAggregationBuilder("agg1").field("num"));
+
+        searchQueryCategorizer.categorize(sourceBuilder);
+
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.termCounter).add(eq(1.0d), any(Tags.class));
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.matchCounter).add(eq(1.0d), any(Tags.class));
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.regexCounter).add(eq(1.0d), any(Tags.class));
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.boolCounter).add(eq(1.0d), any(Tags.class));
+        Mockito.verify(searchQueryCategorizer.searchQueryCounters.aggCounter).add(eq(1.0d));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java b/server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java
new file mode 100644
index 0000000000000..18b814aec61c2
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/query/QueryShapeVisitorTests.java
@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.query;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+import static org.junit.Assert.assertEquals;
+
+public final class QueryShapeVisitorTests extends OpenSearchTestCase {
+    public void testQueryShapeVisitor() {
+        QueryBuilder builder = new BoolQueryBuilder().must(new TermQueryBuilder("foo", "bar"))
+            .filter(new ConstantScoreQueryBuilder(new RangeQueryBuilder("timestamp").from("12345677").to("2345678")))
+            .should(
+                new BoolQueryBuilder().must(new MatchQueryBuilder("text", "this is some text"))
+                    .mustNot(new RegexpQueryBuilder("color", "red.*"))
+            )
+            .must(new TermsQueryBuilder("genre", "action", "drama", "romance"));
+        QueryShapeVisitor shapeVisitor = new QueryShapeVisitor();
+        builder.visit(shapeVisitor);
+        assertEquals(
+            "{\"type\":\"bool\",\"must\"[{\"type\":\"term\"},{\"type\":\"terms\"}],\"filter\"[{\"type\":\"constant_score\",\"filter\"[{\"type\":\"range\"}]}],\"should\"[{\"type\":\"bool\",\"must\"[{\"type\":\"match\"}],\"must_not\"[{\"type\":\"regexp\"}]}]}",
+            shapeVisitor.toJson()
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
index 97c5d23831965..2f9f38d18a064 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
@@ -222,6 +222,7 @@
 import org.opensearch.search.query.QueryPhase;
 import org.opensearch.snapshots.mockstore.MockEventuallyConsistentRepository;
 import org.opensearch.tasks.TaskResourceTrackingService;
+import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.disruption.DisruptableMockTransport;
@@ -2302,7 +2303,8 @@ public void onFailure(final Exception e) {
                             List.of(),
                             client
                         ),
-                        null
+                        null,
+                        NoopMetricsRegistry.INSTANCE
                     )
                 );
                 actions.put(

From 200ad5d28a577877be530ecab507601898025c5c Mon Sep 17 00:00:00 2001
From: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com>
Date: Thu, 19 Oct 2023 19:58:13 -0700
Subject: [PATCH 195/388] Introduce ConcurrentQueryProfiler to profile query
 using concurrent segment search path and support concurrency during rewrite
 and create weight (#10352)

* Fix timer race condition in profile rewrite and create weight for concurrent segment search (#10352)

Signed-off-by: Ticheng Lin 

* Refactor and work on the PR comments (#10352)

Signed-off-by: Ticheng Lin 

---------

Signed-off-by: Ticheng Lin 
---
 CHANGELOG.md                                  |   1 +
 .../search/profile/query/QueryProfilerIT.java | 157 +++++++++++++++++-
 .../opensearch/search/profile/Profilers.java  |   7 +-
 .../org/opensearch/search/profile/Timer.java  |  12 ++
 .../query/AbstractQueryProfileTree.java       |   5 +-
 .../ConcurrentQueryProfileBreakdown.java      |  27 ++-
 .../query/ConcurrentQueryProfiler.java        | 134 +++++++++++++++
 .../search/profile/query/QueryProfiler.java   |  14 +-
 .../ConcurrentQueryProfileBreakdownTests.java |  52 ++++++
 .../query/ConcurrentQueryProfilerTests.java   |  36 ++++
 .../profile/query/QueryProfilerTests.java     |  16 +-
 11 files changed, 438 insertions(+), 23 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java
 create mode 100644 server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 552c277789dd7..5c52c43a35b8d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
 - [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567))
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
+- Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java
index 5f794d2abf878..ef73438114079 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.search.profile.query;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.lucene.tests.util.English;
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.action.search.MultiSearchResponse;
@@ -40,20 +42,23 @@
 import org.opensearch.action.search.SearchType;
 import org.opensearch.action.search.ShardSearchFailure;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.profile.ProfileResult;
 import org.opensearch.search.profile.ProfileShardResult;
 import org.opensearch.search.sort.SortOrder;
-import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.opensearch.search.profile.query.RandomQueryGenerator.randomQueryBuilder;
 import static org.hamcrest.Matchers.emptyOrNullString;
 import static org.hamcrest.Matchers.equalTo;
@@ -61,8 +66,32 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+public class QueryProfilerIT extends ParameterizedOpenSearchIntegTestCase {
+    private final boolean concurrentSearchEnabled;
+    private static final String MAX_PREFIX = "max_";
+    private static final String MIN_PREFIX = "min_";
+    private static final String AVG_PREFIX = "avg_";
+    private static final String TIMING_TYPE_COUNT_SUFFIX = "_count";
+
+    public QueryProfilerIT(Settings settings, boolean concurrentSearchEnabled) {
+        super(settings);
+        this.concurrentSearchEnabled = concurrentSearchEnabled;
+    }
 
-public class QueryProfilerIT extends OpenSearchIntegTestCase {
+    @ParametersFactory
+    public static Collection parameters() {
+        return Arrays.asList(
+            new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build(), false },
+            new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build(), true }
+        );
+    }
+
+    @Override
+    protected Settings featureFlagSettings() {
+        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
+    }
 
     /**
      * This test simply checks to make sure nothing crashes.  Test indexes 100-150 documents,
@@ -229,6 +258,7 @@ public void testSimpleMatch() throws Exception {
                     assertEquals(result.getLuceneDescription(), "field1:one");
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
+                    assertQueryProfileResult(result);
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -271,6 +301,7 @@ public void testBool() throws Exception {
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
                     assertEquals(result.getProfiledChildren().size(), 2);
+                    assertQueryProfileResult(result);
 
                     // Check the children
                     List children = result.getProfiledChildren();
@@ -282,12 +313,14 @@ public void testBool() throws Exception {
                     assertThat(childProfile.getTime(), greaterThan(0L));
                     assertNotNull(childProfile.getTimeBreakdown());
                     assertEquals(childProfile.getProfiledChildren().size(), 0);
+                    assertQueryProfileResult(childProfile);
 
                     childProfile = children.get(1);
                     assertEquals(childProfile.getQueryName(), "TermQuery");
                     assertEquals(childProfile.getLuceneDescription(), "field1:two");
                     assertThat(childProfile.getTime(), greaterThan(0L));
                     assertNotNull(childProfile.getTimeBreakdown());
+                    assertQueryProfileResult(childProfile);
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -330,6 +363,7 @@ public void testEmptyBool() throws Exception {
                     assertNotNull(result.getLuceneDescription());
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
+                    assertQueryProfileResult(result);
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -375,6 +409,7 @@ public void testCollapsingBool() throws Exception {
                     assertNotNull(result.getLuceneDescription());
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
+                    assertQueryProfileResult(result);
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -415,6 +450,90 @@ public void testBoosting() throws Exception {
                     assertNotNull(result.getLuceneDescription());
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
+                    assertQueryProfileResult(result);
+                }
+
+                CollectorResult result = searchProfiles.getCollectorResult();
+                assertThat(result.getName(), is(not(emptyOrNullString())));
+                assertThat(result.getTime(), greaterThan(0L));
+            }
+        }
+    }
+
+    public void testSearchLeafForItsLeavesAndRewriteQuery() throws Exception {
+        createIndex("test");
+        ensureGreen();
+
+        int numDocs = 122;
+        IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+        for (int i = 0; i < numDocs; i++) {
+            docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource("field1", English.intToEnglish(i), "field2", i);
+        }
+
+        List terms = Arrays.asList("zero", "zero", "one");
+
+        indexRandom(true, docs);
+
+        refresh();
+
+        QueryBuilder q = QueryBuilders.boostingQuery(
+            QueryBuilders.idsQuery().addIds(String.valueOf(randomInt()), String.valueOf(randomInt())),
+            QueryBuilders.termsQuery("field1", terms)
+        ).boost(randomFloat()).negativeBoost(randomFloat());
+        logger.info("Query: {}", q);
+
+        SearchResponse resp = client().prepareSearch()
+            .setQuery(q)
+            .setTrackTotalHits(true)
+            .setProfile(true)
+            .setSearchType(SearchType.QUERY_THEN_FETCH)
+            .get();
+
+        assertNotNull("Profile response element should not be null", resp.getProfileResults());
+        assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
+
+        for (Map.Entry shardResult : resp.getProfileResults().entrySet()) {
+            assertThat(shardResult.getValue().getNetworkTime().getInboundNetworkTime(), greaterThanOrEqualTo(0L));
+            assertThat(shardResult.getValue().getNetworkTime().getOutboundNetworkTime(), greaterThanOrEqualTo(0L));
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
+                List results = searchProfiles.getQueryResults();
+                for (ProfileResult result : results) {
+                    assertNotNull(result.getQueryName());
+                    assertNotNull(result.getLuceneDescription());
+                    assertThat(result.getTime(), greaterThan(0L));
+                    Map breakdown = result.getTimeBreakdown();
+                    Long maxSliceTime = result.getMaxSliceTime();
+                    Long minSliceTime = result.getMinSliceTime();
+                    Long avgSliceTime = result.getAvgSliceTime();
+                    if (concurrentSearchEnabled && results.get(0).equals(result)) {
+                        assertNotNull(maxSliceTime);
+                        assertNotNull(minSliceTime);
+                        assertNotNull(avgSliceTime);
+                        assertThat(breakdown.size(), equalTo(66));
+                        for (QueryTimingType queryTimingType : QueryTimingType.values()) {
+                            if (queryTimingType != QueryTimingType.CREATE_WEIGHT) {
+                                String maxTimingType = MAX_PREFIX + queryTimingType;
+                                String minTimingType = MIN_PREFIX + queryTimingType;
+                                String avgTimingType = AVG_PREFIX + queryTimingType;
+                                assertNotNull(breakdown.get(maxTimingType));
+                                assertNotNull(breakdown.get(minTimingType));
+                                assertNotNull(breakdown.get(avgTimingType));
+                                assertNotNull(breakdown.get(maxTimingType + TIMING_TYPE_COUNT_SUFFIX));
+                                assertNotNull(breakdown.get(minTimingType + TIMING_TYPE_COUNT_SUFFIX));
+                                assertNotNull(breakdown.get(avgTimingType + TIMING_TYPE_COUNT_SUFFIX));
+                            }
+                        }
+                    } else if (concurrentSearchEnabled) {
+                        assertThat(maxSliceTime, equalTo(0L));
+                        assertThat(minSliceTime, equalTo(0L));
+                        assertThat(avgSliceTime, equalTo(0L));
+                        assertThat(breakdown.size(), equalTo(27));
+                    } else {
+                        assertThat(maxSliceTime, is(nullValue()));
+                        assertThat(minSliceTime, is(nullValue()));
+                        assertThat(avgSliceTime, is(nullValue()));
+                        assertThat(breakdown.size(), equalTo(27));
+                    }
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -455,6 +574,7 @@ public void testDisMaxRange() throws Exception {
                     assertNotNull(result.getLuceneDescription());
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
+                    assertQueryProfileResult(result);
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -494,6 +614,7 @@ public void testRange() throws Exception {
                     assertNotNull(result.getLuceneDescription());
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
+                    assertQueryProfileResult(result);
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -547,6 +668,7 @@ public void testPhrase() throws Exception {
                     assertNotNull(result.getLuceneDescription());
                     assertThat(result.getTime(), greaterThan(0L));
                     assertNotNull(result.getTimeBreakdown());
+                    assertQueryProfileResult(result);
                 }
 
                 CollectorResult result = searchProfiles.getCollectorResult();
@@ -579,4 +701,35 @@ public void testNoProfile() throws Exception {
         assertThat("Profile response element should be an empty map", resp.getProfileResults().size(), equalTo(0));
     }
 
+    private void assertQueryProfileResult(ProfileResult result) {
+        Map breakdown = result.getTimeBreakdown();
+        Long maxSliceTime = result.getMaxSliceTime();
+        Long minSliceTime = result.getMinSliceTime();
+        Long avgSliceTime = result.getAvgSliceTime();
+        if (concurrentSearchEnabled) {
+            assertNotNull(maxSliceTime);
+            assertNotNull(minSliceTime);
+            assertNotNull(avgSliceTime);
+            assertThat(breakdown.size(), equalTo(66));
+            for (QueryTimingType queryTimingType : QueryTimingType.values()) {
+                if (queryTimingType != QueryTimingType.CREATE_WEIGHT) {
+                    String maxTimingType = MAX_PREFIX + queryTimingType;
+                    String minTimingType = MIN_PREFIX + queryTimingType;
+                    String avgTimingType = AVG_PREFIX + queryTimingType;
+                    assertNotNull(breakdown.get(maxTimingType));
+                    assertNotNull(breakdown.get(minTimingType));
+                    assertNotNull(breakdown.get(avgTimingType));
+                    assertNotNull(breakdown.get(maxTimingType + TIMING_TYPE_COUNT_SUFFIX));
+                    assertNotNull(breakdown.get(minTimingType + TIMING_TYPE_COUNT_SUFFIX));
+                    assertNotNull(breakdown.get(avgTimingType + TIMING_TYPE_COUNT_SUFFIX));
+                }
+            }
+        } else {
+            assertThat(maxSliceTime, is(nullValue()));
+            assertThat(minSliceTime, is(nullValue()));
+            assertThat(avgSliceTime, is(nullValue()));
+            assertThat(breakdown.size(), equalTo(27));
+        }
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/search/profile/Profilers.java b/server/src/main/java/org/opensearch/search/profile/Profilers.java
index 8e87c7ff4acd4..68cf05c988b5b 100644
--- a/server/src/main/java/org/opensearch/search/profile/Profilers.java
+++ b/server/src/main/java/org/opensearch/search/profile/Profilers.java
@@ -35,6 +35,9 @@
 import org.opensearch.search.internal.ContextIndexSearcher;
 import org.opensearch.search.profile.aggregation.AggregationProfiler;
 import org.opensearch.search.profile.aggregation.ConcurrentAggregationProfiler;
+import org.opensearch.search.profile.query.ConcurrentQueryProfileTree;
+import org.opensearch.search.profile.query.ConcurrentQueryProfiler;
+import org.opensearch.search.profile.query.InternalQueryProfileTree;
 import org.opensearch.search.profile.query.QueryProfiler;
 
 import java.util.ArrayList;
@@ -64,7 +67,9 @@ public Profilers(ContextIndexSearcher searcher, boolean isConcurrentSegmentSearc
 
     /** Switch to a new profile. */
     public QueryProfiler addQueryProfiler() {
-        QueryProfiler profiler = new QueryProfiler(isConcurrentSegmentSearchEnabled);
+        QueryProfiler profiler = isConcurrentSegmentSearchEnabled
+            ? new ConcurrentQueryProfiler(new ConcurrentQueryProfileTree())
+            : new QueryProfiler(new InternalQueryProfileTree());
         searcher.setProfiler(profiler);
         queryProfilers.add(profiler);
         return profiler;
diff --git a/server/src/main/java/org/opensearch/search/profile/Timer.java b/server/src/main/java/org/opensearch/search/profile/Timer.java
index 172762cabeb6a..864c689cf7fa0 100644
--- a/server/src/main/java/org/opensearch/search/profile/Timer.java
+++ b/server/src/main/java/org/opensearch/search/profile/Timer.java
@@ -53,6 +53,18 @@ public class Timer {
     private boolean doTiming;
     private long timing, count, lastCount, start, earliestTimerStartTime;
 
+    public Timer() {
+        this(0, 0, 0, 0, 0);
+    }
+
+    public Timer(long timing, long count, long lastCount, long start, long earliestTimerStartTime) {
+        this.timing = timing;
+        this.count = count;
+        this.lastCount = lastCount;
+        this.start = start;
+        this.earliestTimerStartTime = earliestTimerStartTime;
+    }
+
     /** pkg-private for testing */
     long nanoTime() {
         return System.nanoTime();
diff --git a/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java b/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java
index 8e825def13f5d..2f5d632ee2d87 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/AbstractQueryProfileTree.java
@@ -54,14 +54,11 @@ public void startRewriteTime() {
      * startRewriteTime() must be called for a particular context prior to calling
      * stopAndAddRewriteTime(), otherwise the elapsed time will be negative and
      * nonsensical
-     *
-     * @return          The elapsed time
      */
-    public long stopAndAddRewriteTime() {
+    public void stopAndAddRewriteTime() {
         long time = Math.max(1, System.nanoTime() - rewriteScratch);
         rewriteTime += time;
         rewriteScratch = 0;
-        return time;
     }
 
     public long getRewriteTime() {
diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
index e567fdd2d436c..59ef01f9f947a 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
@@ -70,7 +70,7 @@ public Map toBreakdownMap() {
         );
         final long createWeightTime = topLevelBreakdownMapWithWeightTime.get(QueryTimingType.CREATE_WEIGHT.toString());
 
-        if (sliceCollectorsToLeaves.isEmpty() || contexts.isEmpty()) {
+        if (contexts.isEmpty()) {
             // If there are no leaf contexts, then return the default concurrent query level breakdown, which will include the
             // create_weight time/count
             queryNodeTime = createWeightTime;
@@ -78,6 +78,21 @@ public Map toBreakdownMap() {
             minSliceNodeTime = 0L;
             avgSliceNodeTime = 0L;
             return buildDefaultQueryBreakdownMap(createWeightTime);
+        } else if (sliceCollectorsToLeaves.isEmpty()) {
+            // This will happen when each slice executes search leaf for its leaves and query is rewritten for the leaf being searched. It
+            // creates a new weight and breakdown map for each rewritten query. This new breakdown map captures the timing information for
+            // the new rewritten query. The sliceCollectorsToLeaves is empty because this breakdown for rewritten query gets created later
+            // in search leaf path which doesn't have collector. Also, this is not needed since this breakdown is per leaf and there is no
+            // concurrency involved. An empty sliceCollectorsToLeaves could also happen in the case of early termination.
+            AbstractProfileBreakdown breakdown = contexts.values().iterator().next();
+            queryNodeTime = breakdown.toNodeTime() + createWeightTime;
+            maxSliceNodeTime = 0L;
+            minSliceNodeTime = 0L;
+            avgSliceNodeTime = 0L;
+            Map queryBreakdownMap = new HashMap<>(breakdown.toBreakdownMap());
+            queryBreakdownMap.put(QueryTimingType.CREATE_WEIGHT.toString(), createWeightTime);
+            queryBreakdownMap.put(QueryTimingType.CREATE_WEIGHT + TIMING_TYPE_COUNT_SUFFIX, 1L);
+            return queryBreakdownMap;
         }
 
         // first create the slice level breakdowns
@@ -191,10 +206,12 @@ Map> buildSliceLevelBreakdown() {
                 }
                 // compute sliceMaxEndTime as max of sliceEndTime across all timing types
                 sliceMaxEndTime = Math.max(sliceMaxEndTime, currentSliceBreakdown.getOrDefault(timingTypeSliceEndTimeKey, Long.MIN_VALUE));
-                sliceMinStartTime = Math.min(
-                    sliceMinStartTime,
-                    currentSliceBreakdown.getOrDefault(timingTypeSliceStartTimeKey, Long.MAX_VALUE)
-                );
+                long currentSliceStartTime = currentSliceBreakdown.getOrDefault(timingTypeSliceStartTimeKey, Long.MAX_VALUE);
+                if (currentSliceStartTime == 0L) {
+                    // The timer for the current timing type never starts, so we continue here
+                    continue;
+                }
+                sliceMinStartTime = Math.min(sliceMinStartTime, currentSliceStartTime);
                 // compute total time for each timing type at slice level using sliceEndTime and sliceStartTime
                 currentSliceBreakdown.put(
                     timingType.toString(),
diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java
new file mode 100644
index 0000000000000..42bf23bb13fbe
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfiler.java
@@ -0,0 +1,134 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.query;
+
+import org.apache.lucene.search.Query;
+import org.opensearch.search.profile.ContextualProfileBreakdown;
+import org.opensearch.search.profile.ProfileResult;
+import org.opensearch.search.profile.Timer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * This class acts as a thread-local storage for profiling a query with concurrent execution
+ *
+ * @opensearch.internal
+ */
+public final class ConcurrentQueryProfiler extends QueryProfiler {
+
+    private final Map threadToProfileTree;
+    // The LinkedList does not need to be thread safe, as the map associates thread IDs with LinkedList, and only
+    // one thread will access the LinkedList at a time.
+    private final Map> threadToRewriteTimers;
+
+    public ConcurrentQueryProfiler(AbstractQueryProfileTree profileTree) {
+        super(profileTree);
+        long threadId = getCurrentThreadId();
+        // We utilize LinkedHashMap to preserve the insertion order of the profiled queries
+        threadToProfileTree = Collections.synchronizedMap(new LinkedHashMap<>());
+        threadToProfileTree.put(threadId, (ConcurrentQueryProfileTree) profileTree);
+        threadToRewriteTimers = new ConcurrentHashMap<>();
+        threadToRewriteTimers.put(threadId, new LinkedList<>());
+    }
+
+    @Override
+    public ContextualProfileBreakdown getQueryBreakdown(Query query) {
+        ConcurrentQueryProfileTree profileTree = threadToProfileTree.computeIfAbsent(
+            getCurrentThreadId(),
+            k -> new ConcurrentQueryProfileTree()
+        );
+        return profileTree.getProfileBreakdown(query);
+    }
+
+    /**
+     * Removes the last (e.g. most recent) element on ConcurrentQueryProfileTree stack.
+     */
+    @Override
+    public void pollLastElement() {
+        ConcurrentQueryProfileTree concurrentProfileTree = threadToProfileTree.get(getCurrentThreadId());
+        if (concurrentProfileTree != null) {
+            concurrentProfileTree.pollLast();
+        }
+    }
+
+    /**
+     * @return a hierarchical representation of the profiled tree
+     */
+    @Override
+    public List getTree() {
+        List profileResults = new ArrayList<>();
+        for (Map.Entry profile : threadToProfileTree.entrySet()) {
+            profileResults.addAll(profile.getValue().getTree());
+        }
+        return profileResults;
+    }
+
+    /**
+     * Begin timing the rewrite phase of a request
+     */
+    @Override
+    public void startRewriteTime() {
+        Timer rewriteTimer = new Timer();
+        threadToRewriteTimers.computeIfAbsent(getCurrentThreadId(), k -> new LinkedList<>()).add(rewriteTimer);
+        rewriteTimer.start();
+    }
+
+    /**
+     * Stop recording the current rewrite timer
+     */
+    public void stopAndAddRewriteTime() {
+        Timer rewriteTimer = threadToRewriteTimers.get(getCurrentThreadId()).getLast();
+        rewriteTimer.stop();
+    }
+
+    /**
+     * @return total time taken to rewrite all queries in this concurrent query profiler
+     */
+    @Override
+    public long getRewriteTime() {
+        long totalRewriteTime = 0L;
+        List rewriteTimers = new LinkedList<>();
+        threadToRewriteTimers.values().forEach(rewriteTimers::addAll);
+        LinkedList mergedIntervals = mergeRewriteTimeIntervals(rewriteTimers);
+        for (long[] interval : mergedIntervals) {
+            totalRewriteTime += interval[1] - interval[0];
+        }
+        return totalRewriteTime;
+    }
+
+    // package private for unit testing
+    LinkedList mergeRewriteTimeIntervals(List timers) {
+        LinkedList mergedIntervals = new LinkedList<>();
+        timers.sort(Comparator.comparingLong(Timer::getEarliestTimerStartTime));
+        for (Timer timer : timers) {
+            long startTime = timer.getEarliestTimerStartTime();
+            long endTime = startTime + timer.getApproximateTiming();
+            if (mergedIntervals.isEmpty() || mergedIntervals.getLast()[1] < startTime) {
+                long[] interval = new long[2];
+                interval[0] = startTime;
+                interval[1] = endTime;
+                mergedIntervals.add(interval);
+            } else {
+                mergedIntervals.getLast()[1] = Math.max(mergedIntervals.getLast()[1], endTime);
+            }
+        }
+        return mergedIntervals;
+    }
+
+    private long getCurrentThreadId() {
+        return Thread.currentThread().getId();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java b/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java
index a80ce1c658081..332c4b3551450 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/QueryProfiler.java
@@ -51,15 +51,15 @@
  *
  * @opensearch.internal
  */
-public final class QueryProfiler extends AbstractProfiler, Query> {
+public class QueryProfiler extends AbstractProfiler, Query> {
 
     /**
      * The root Collector used in the search
      */
     private InternalProfileComponent collector;
 
-    public QueryProfiler(boolean concurrent) {
-        super(concurrent ? new ConcurrentQueryProfileTree() : new InternalQueryProfileTree());
+    public QueryProfiler(AbstractQueryProfileTree profileTree) {
+        super(profileTree);
     }
 
     /** Set the collector that is associated with this profiler. */
@@ -81,14 +81,14 @@ public void startRewriteTime() {
     /**
      * Stop recording the current rewrite and add it's time to the total tally, returning the
      * cumulative time so far.
-     *
-     * @return cumulative rewrite time
      */
-    public long stopAndAddRewriteTime() {
-        return ((AbstractQueryProfileTree) profileTree).stopAndAddRewriteTime();
+    public void stopAndAddRewriteTime() {
+        ((AbstractQueryProfileTree) profileTree).stopAndAddRewriteTime();
     }
 
     /**
+     * The rewriting process is complex and hard to display because queries can undergo significant changes.
+     * Instead of showing intermediate results, we display the cumulative time for the non-concurrent search case.
      * @return total time taken to rewrite all queries in this profile
      */
     public long getRewriteTime() {
diff --git a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java
index f29ba3b0cea07..db14eb90ef839 100644
--- a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java
+++ b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdownTests.java
@@ -333,6 +333,58 @@ public void testBreakDownMapWithMultipleSlicesAndOneSliceWithNoLeafContext() thr
         directory.close();
     }
 
+    public void testOneLeafContextWithEmptySliceCollectorsToLeaves() throws Exception {
+        final DirectoryReader directoryReader = getDirectoryReader(1);
+        final Directory directory = directoryReader.directory();
+        final long createWeightEarliestStartTime = createWeightTimer.getEarliestTimerStartTime();
+        final long createWeightEndTime = createWeightEarliestStartTime + createWeightTimer.getApproximateTiming();
+        final Map leafProfileBreakdownMap_1 = getLeafBreakdownMap(createWeightEndTime + 10, 10, 1);
+        final AbstractProfileBreakdown leafProfileBreakdown_1 = new TestQueryProfileBreakdown(
+            QueryTimingType.class,
+            leafProfileBreakdownMap_1
+        );
+        testQueryProfileBreakdown.getContexts().put(directoryReader.leaves().get(0), leafProfileBreakdown_1);
+        final Map queryBreakDownMap = testQueryProfileBreakdown.toBreakdownMap();
+        assertFalse(queryBreakDownMap == null || queryBreakDownMap.isEmpty());
+        assertEquals(26, queryBreakDownMap.size());
+        for (QueryTimingType queryTimingType : QueryTimingType.values()) {
+            String timingTypeKey = queryTimingType.toString();
+            String timingTypeCountKey = queryTimingType + TIMING_TYPE_COUNT_SUFFIX;
+
+            if (queryTimingType.equals(QueryTimingType.CREATE_WEIGHT)) {
+                final long createWeightTime = queryBreakDownMap.get(timingTypeKey);
+                assertEquals(createWeightTimer.getApproximateTiming(), createWeightTime);
+                assertEquals(1, (long) queryBreakDownMap.get(timingTypeCountKey));
+                // verify there is no min/max/avg for weight type stats
+                assertFalse(
+                    queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeKey)
+                        || queryBreakDownMap.containsKey(MIN_PREFIX + timingTypeKey)
+                        || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeKey)
+                        || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeCountKey)
+                        || queryBreakDownMap.containsKey(MIN_PREFIX + timingTypeCountKey)
+                        || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeCountKey)
+                );
+                continue;
+            }
+            assertNotNull(queryBreakDownMap.get(timingTypeKey));
+            assertNotNull(queryBreakDownMap.get(timingTypeCountKey));
+            // verify there is no min/max/avg for current breakdown type stats
+            assertFalse(
+                queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeKey)
+                    || queryBreakDownMap.containsKey(MIN_PREFIX + timingTypeKey)
+                    || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeKey)
+                    || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.MAX_PREFIX + timingTypeCountKey)
+                    || queryBreakDownMap.containsKey(MIN_PREFIX + timingTypeCountKey)
+                    || queryBreakDownMap.containsKey(ConcurrentQueryProfileBreakdown.AVG_PREFIX + timingTypeCountKey)
+            );
+        }
+        assertEquals(0, testQueryProfileBreakdown.getMaxSliceNodeTime());
+        assertEquals(0, testQueryProfileBreakdown.getMinSliceNodeTime());
+        assertEquals(0, testQueryProfileBreakdown.getAvgSliceNodeTime());
+        directoryReader.close();
+        directory.close();
+    }
+
     private Map getLeafBreakdownMap(long startTime, long timeTaken, long count) {
         Map leafBreakDownMap = new HashMap<>();
         for (QueryTimingType timingType : QueryTimingType.values()) {
diff --git a/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java
new file mode 100644
index 0000000000000..736bbcdd9e8dd
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/profile/query/ConcurrentQueryProfilerTests.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.profile.query;
+
+import org.opensearch.search.profile.Timer;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ConcurrentQueryProfilerTests extends OpenSearchTestCase {
+
+    public void testMergeRewriteTimeIntervals() {
+        ConcurrentQueryProfiler profiler = new ConcurrentQueryProfiler(new ConcurrentQueryProfileTree());
+        List timers = new LinkedList<>();
+        timers.add(new Timer(217134L, 1L, 1L, 0L, 553074511206907L));
+        timers.add(new Timer(228954L, 1L, 1L, 0L, 553074509287335L));
+        timers.add(new Timer(228954L, 1L, 1L, 0L, 553074509287336L));
+        LinkedList mergedIntervals = profiler.mergeRewriteTimeIntervals(timers);
+        assertThat(mergedIntervals.size(), equalTo(2));
+        long[] interval = mergedIntervals.get(0);
+        assertThat(interval[0], equalTo(553074509287335L));
+        assertThat(interval[1], equalTo(553074509516290L));
+        interval = mergedIntervals.get(1);
+        assertThat(interval[0], equalTo(553074511206907L));
+        assertThat(interval[1], equalTo(553074511424041L));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java
index 64a440b85eb10..481a224f2ff0e 100644
--- a/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java
+++ b/server/src/test/java/org/opensearch/search/profile/query/QueryProfilerTests.java
@@ -161,7 +161,9 @@ public void tearDown() throws Exception {
     }
 
     public void testBasic() throws IOException {
-        QueryProfiler profiler = new QueryProfiler(executor != null);
+        QueryProfiler profiler = executor != null
+            ? new ConcurrentQueryProfiler(new ConcurrentQueryProfileTree())
+            : new QueryProfiler(new InternalQueryProfileTree());
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.search(query, 1);
@@ -228,7 +230,9 @@ public void testBasic() throws IOException {
     }
 
     public void testNoScoring() throws IOException {
-        QueryProfiler profiler = new QueryProfiler(executor != null);
+        QueryProfiler profiler = executor != null
+            ? new ConcurrentQueryProfiler(new ConcurrentQueryProfileTree())
+            : new QueryProfiler(new InternalQueryProfileTree());
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.search(query, 1, Sort.INDEXORDER); // scores are not needed
@@ -295,7 +299,9 @@ public void testNoScoring() throws IOException {
     }
 
     public void testUseIndexStats() throws IOException {
-        QueryProfiler profiler = new QueryProfiler(executor != null);
+        QueryProfiler profiler = executor != null
+            ? new ConcurrentQueryProfiler(new ConcurrentQueryProfileTree())
+            : new QueryProfiler(new InternalQueryProfileTree());
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.count(query); // will use index stats
@@ -309,7 +315,9 @@ public void testUseIndexStats() throws IOException {
     }
 
     public void testApproximations() throws IOException {
-        QueryProfiler profiler = new QueryProfiler(executor != null);
+        QueryProfiler profiler = executor != null
+            ? new ConcurrentQueryProfiler(new ConcurrentQueryProfileTree())
+            : new QueryProfiler(new InternalQueryProfileTree());
         searcher.setProfiler(profiler);
         Query query = new RandomApproximationQuery(new TermQuery(new Term("foo", "bar")), random());
         searcher.count(query);

From 41a12e28fff58c7262bfeb725c8a31e5ffa266f3 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Thu, 19 Oct 2023 20:33:32 -0700
Subject: [PATCH 196/388] Make RemoteStoreReplicationSource#getSegmentFiles
 asynchronous (#10725)

* Make RemoteStoreReplicationSource#getSegmentFiles asynchronous

Also make the remote store download process cancellable in case the
replication event is canceled.

Signed-off-by: Andrew Ross 

* Add ITs ensuring segRep targets are cleaned up on cancellation during metadata and segment fetch steps.

Signed-off-by: Marc Handalian 

* Wrap metadata fetch in cancellableThreads.executeIO

Signed-off-by: Marc Handalian 

* self review

Signed-off-by: Marc Handalian 

* spotless

Signed-off-by: Marc Handalian 

* Add missing node settings when bootstrapping nodes in tests.

Signed-off-by: Marc Handalian 

---------

Signed-off-by: Andrew Ross 
Signed-off-by: Marc Handalian 
Co-authored-by: Andrew Ross 
---
 ...emoteStoreMockRepositoryIntegTestCase.java |   5 +
 ...plicationUsingRemoteStoreDisruptionIT.java | 133 ++++++++++++++++++
 .../store/RemoteStoreFileDownloader.java      |  72 ++++++----
 .../RemoteStoreReplicationSource.java         |  68 +++++----
 .../replication/common/ReplicationTarget.java |  11 +-
 .../store/RemoteStoreFileDownloaderTests.java | 125 ++++++++++++++--
 .../AbstractSnapshotIntegTestCase.java        |   6 +
 .../snapshots/mockstore/MockRepository.java   |  15 ++
 8 files changed, 373 insertions(+), 62 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreDisruptionIT.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java
index 2053800504c89..8166c0008ed83 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java
@@ -114,6 +114,10 @@ protected void cleanupRepo() {
     }
 
     protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) {
+        return setup(repoLocation, ioFailureRate, skipExceptionBlobList, maxFailure, 0);
+    }
+
+    protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure, int replicaCount) {
         // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in
         /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the
         // repository creation can happen without failure.
@@ -128,6 +132,7 @@ protected String setup(Path repoLocation, double ioFailureRate, String skipExcep
 
         internalCluster().startClusterManagerOnlyNode(settings.build());
         String dataNodeName = internalCluster().startDataOnlyNode(settings.build());
+        internalCluster().startDataOnlyNodes(replicaCount, settings.build());
         createIndex(INDEX_NAME);
         logger.info("--> Created index={}", INDEX_NAME);
         ensureYellowAndNoInitializingShards(INDEX_NAME);
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreDisruptionIT.java
new file mode 100644
index 0000000000000..b7b3f1d14f422
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreDisruptionIT.java
@@ -0,0 +1,133 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotestore;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.index.Index;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.replication.SegmentReplicationState;
+import org.opensearch.indices.replication.SegmentReplicationTarget;
+import org.opensearch.indices.replication.SegmentReplicationTargetService;
+import org.opensearch.indices.replication.common.ReplicationCollection;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class runs tests with remote store + segRep while blocking file downloads
+ */
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class SegmentReplicationUsingRemoteStoreDisruptionIT extends AbstractRemoteStoreMockRepositoryIntegTestCase {
+
+    @Override
+    public Settings indexSettings() {
+        return remoteStoreIndexSettings(1);
+    }
+
+    @Override
+    protected boolean addMockInternalEngine() {
+        return false;
+    }
+
+    public void testCancelReplicationWhileSyncingSegments() throws Exception {
+        Path location = randomRepoPath().toAbsolutePath();
+        setup(location, 0d, "metadata", Long.MAX_VALUE, 1);
+
+        final Set dataNodeNames = internalCluster().getDataNodeNames();
+        final String replicaNode = getNode(dataNodeNames, false);
+        final String primaryNode = getNode(dataNodeNames, true);
+
+        SegmentReplicationTargetService targetService = internalCluster().getInstance(SegmentReplicationTargetService.class, replicaNode);
+        ensureGreen(INDEX_NAME);
+        blockNodeOnAnySegmentFile(REPOSITORY_NAME, replicaNode);
+        final IndexShard indexShard = getIndexShard(replicaNode, INDEX_NAME);
+        indexSingleDoc();
+        refresh(INDEX_NAME);
+        waitForBlock(replicaNode, REPOSITORY_NAME, TimeValue.timeValueSeconds(10));
+        final SegmentReplicationState state = targetService.getOngoingEventSegmentReplicationState(indexShard.shardId());
+        assertEquals(SegmentReplicationState.Stage.GET_FILES, state.getStage());
+        ReplicationCollection.ReplicationRef segmentReplicationTargetReplicationRef = targetService.get(
+            state.getReplicationId()
+        );
+        final SegmentReplicationTarget segmentReplicationTarget = segmentReplicationTargetReplicationRef.get();
+        // close the target ref here otherwise it will hold a refcount
+        segmentReplicationTargetReplicationRef.close();
+        assertNotNull(segmentReplicationTarget);
+        assertTrue(segmentReplicationTarget.refCount() > 0);
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
+        assertBusy(() -> {
+            assertTrue(indexShard.routingEntry().primary());
+            assertNull(targetService.getOngoingEventSegmentReplicationState(indexShard.shardId()));
+            assertEquals("Target should be closed", 0, segmentReplicationTarget.refCount());
+        });
+        unblockNode(REPOSITORY_NAME, replicaNode);
+        cleanupRepo();
+    }
+
+    public void testCancelReplicationWhileFetchingMetadata() throws Exception {
+        Path location = randomRepoPath().toAbsolutePath();
+        setup(location, 0d, "metadata", Long.MAX_VALUE, 1);
+
+        final Set dataNodeNames = internalCluster().getDataNodeNames();
+        final String replicaNode = getNode(dataNodeNames, false);
+        final String primaryNode = getNode(dataNodeNames, true);
+
+        SegmentReplicationTargetService targetService = internalCluster().getInstance(SegmentReplicationTargetService.class, replicaNode);
+        ensureGreen(INDEX_NAME);
+        blockNodeOnAnyFiles(REPOSITORY_NAME, replicaNode);
+        final IndexShard indexShard = getIndexShard(replicaNode, INDEX_NAME);
+        indexSingleDoc();
+        refresh(INDEX_NAME);
+        waitForBlock(replicaNode, REPOSITORY_NAME, TimeValue.timeValueSeconds(10));
+        final SegmentReplicationState state = targetService.getOngoingEventSegmentReplicationState(indexShard.shardId());
+        assertEquals(SegmentReplicationState.Stage.GET_CHECKPOINT_INFO, state.getStage());
+        ReplicationCollection.ReplicationRef segmentReplicationTargetReplicationRef = targetService.get(
+            state.getReplicationId()
+        );
+        final SegmentReplicationTarget segmentReplicationTarget = segmentReplicationTargetReplicationRef.get();
+        // close the target ref here otherwise it will hold a refcount
+        segmentReplicationTargetReplicationRef.close();
+        assertNotNull(segmentReplicationTarget);
+        assertTrue(segmentReplicationTarget.refCount() > 0);
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
+        assertBusy(() -> {
+            assertTrue(indexShard.routingEntry().primary());
+            assertNull(targetService.getOngoingEventSegmentReplicationState(indexShard.shardId()));
+            assertEquals("Target should be closed", 0, segmentReplicationTarget.refCount());
+        });
+        unblockNode(REPOSITORY_NAME, replicaNode);
+        cleanupRepo();
+    }
+
+    private String getNode(Set dataNodeNames, boolean primary) {
+        assertEquals(2, dataNodeNames.size());
+        for (String name : dataNodeNames) {
+            final IndexShard indexShard = getIndexShard(name, INDEX_NAME);
+            if (indexShard.routingEntry().primary() == primary) {
+                return name;
+            }
+        }
+        return null;
+    }
+
+    private IndexShard getIndexShard(String node, String indexName) {
+        final Index index = resolveIndex(indexName);
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
+        IndexService indexService = indicesService.indexService(index);
+        assertNotNull(indexService);
+        final Optional shardId = indexService.shardIds().stream().findFirst();
+        return shardId.map(indexService::getShard).orElse(null);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java b/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java
index 4fc721f2b96b5..727c57afd289b 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java
@@ -16,7 +16,7 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.annotation.InternalApi;
 import org.opensearch.common.logging.Loggers;
-import org.opensearch.common.util.concurrent.UncategorizedExecutionException;
+import org.opensearch.common.util.CancellableThreads;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.indices.recovery.RecoverySettings;
@@ -51,9 +51,16 @@ public RemoteStoreFileDownloader(ShardId shardId, ThreadPool threadPool, Recover
      * @param source The remote directory to copy segment files from
      * @param destination The local directory to copy segment files to
      * @param toDownloadSegments The list of segment files to download
+     * @param listener Callback listener to be notified upon completion
      */
-    public void download(Directory source, Directory destination, Collection toDownloadSegments) throws IOException {
-        downloadInternal(source, destination, null, toDownloadSegments, () -> {});
+    public void downloadAsync(
+        CancellableThreads cancellableThreads,
+        Directory source,
+        Directory destination,
+        Collection toDownloadSegments,
+        ActionListener listener
+    ) {
+        downloadInternal(cancellableThreads, source, destination, null, toDownloadSegments, () -> {}, listener);
     }
 
     /**
@@ -74,17 +81,37 @@ public void download(
         Directory secondDestination,
         Collection toDownloadSegments,
         Runnable onFileCompletion
-    ) throws IOException {
-        downloadInternal(source, destination, secondDestination, toDownloadSegments, onFileCompletion);
+    ) throws InterruptedException, IOException {
+        final CancellableThreads cancellableThreads = new CancellableThreads();
+        final PlainActionFuture listener = PlainActionFuture.newFuture();
+        downloadInternal(cancellableThreads, source, destination, secondDestination, toDownloadSegments, onFileCompletion, listener);
+        try {
+            listener.get();
+        } catch (ExecutionException e) {
+            if (e.getCause() instanceof RuntimeException) {
+                throw (RuntimeException) e.getCause();
+            } else if (e.getCause() instanceof IOException) {
+                throw (IOException) e.getCause();
+            }
+            throw new RuntimeException(e);
+        } catch (InterruptedException e) {
+            // If the blocking call on the PlainActionFuture itself is interrupted, then we must
+            // cancel the asynchronous work we were waiting on
+            cancellableThreads.cancel(e.getMessage());
+            Thread.currentThread().interrupt();
+            throw e;
+        }
     }
 
     private void downloadInternal(
+        CancellableThreads cancellableThreads,
         Directory source,
         Directory destination,
         @Nullable Directory secondDestination,
         Collection toDownloadSegments,
-        Runnable onFileCompletion
-    ) throws IOException {
+        Runnable onFileCompletion,
+        ActionListener listener
+    ) {
         final Queue queue = new ConcurrentLinkedQueue<>(toDownloadSegments);
         // Choose the minimum of:
         // - number of files to download
@@ -95,25 +122,14 @@ private void downloadInternal(
             Math.min(threadPool.info(ThreadPool.Names.REMOTE_RECOVERY).getMax(), recoverySettings.getMaxConcurrentRemoteStoreStreams())
         );
         logger.trace("Starting download of {} files with {} threads", queue.size(), threads);
-        final PlainActionFuture> listener = PlainActionFuture.newFuture();
-        final ActionListener allFilesListener = new GroupedActionListener<>(listener, threads);
+        final ActionListener allFilesListener = new GroupedActionListener<>(ActionListener.map(listener, r -> null), threads);
         for (int i = 0; i < threads; i++) {
-            copyOneFile(source, destination, secondDestination, queue, onFileCompletion, allFilesListener);
-        }
-        try {
-            listener.actionGet();
-        } catch (UncategorizedExecutionException e) {
-            // Any IOException will be double-wrapped so dig it out and throw it
-            if (e.getCause() instanceof ExecutionException) {
-                if (e.getCause().getCause() instanceof IOException) {
-                    throw (IOException) e.getCause().getCause();
-                }
-            }
-            throw e;
+            copyOneFile(cancellableThreads, source, destination, secondDestination, queue, onFileCompletion, allFilesListener);
         }
     }
 
     private void copyOneFile(
+        CancellableThreads cancellableThreads,
         Directory source,
         Directory destination,
         @Nullable Directory secondDestination,
@@ -129,18 +145,20 @@ private void copyOneFile(
             threadPool.executor(ThreadPool.Names.REMOTE_RECOVERY).submit(() -> {
                 logger.trace("Downloading file {}", file);
                 try {
-                    destination.copyFrom(source, file, file, IOContext.DEFAULT);
-                    onFileCompletion.run();
-                    if (secondDestination != null) {
-                        secondDestination.copyFrom(destination, file, file, IOContext.DEFAULT);
-                    }
+                    cancellableThreads.executeIO(() -> {
+                        destination.copyFrom(source, file, file, IOContext.DEFAULT);
+                        onFileCompletion.run();
+                        if (secondDestination != null) {
+                            secondDestination.copyFrom(destination, file, file, IOContext.DEFAULT);
+                        }
+                    });
                 } catch (Exception e) {
                     // Clear the queue to stop any future processing, report the failure, then return
                     queue.clear();
                     listener.onFailure(e);
                     return;
                 }
-                copyOneFile(source, destination, secondDestination, queue, onFileCompletion, listener);
+                copyOneFile(cancellableThreads, source, destination, secondDestination, queue, onFileCompletion, listener);
             });
         }
     }
diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java
index 12eabf1e6554f..b06b3e0497cf7 100644
--- a/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java
+++ b/server/src/main/java/org/opensearch/indices/replication/RemoteStoreReplicationSource.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.util.Version;
 import org.opensearch.common.concurrent.GatedCloseable;
+import org.opensearch.common.util.CancellableThreads;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardState;
@@ -24,11 +25,14 @@
 import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
@@ -43,6 +47,7 @@ public class RemoteStoreReplicationSource implements SegmentReplicationSource {
 
     private final IndexShard indexShard;
     private final RemoteSegmentStoreDirectory remoteDirectory;
+    private final CancellableThreads cancellableThreads = new CancellableThreads();
 
     public RemoteStoreReplicationSource(IndexShard indexShard) {
         this.indexShard = indexShard;
@@ -61,7 +66,7 @@ public void getCheckpointMetadata(
         // TODO: Need to figure out a way to pass this information for segment metadata via remote store.
         try (final GatedCloseable segmentInfosSnapshot = indexShard.getSegmentInfosSnapshot()) {
             final Version version = segmentInfosSnapshot.get().getCommitLuceneVersion();
-            RemoteSegmentMetadata mdFile = remoteDirectory.init();
+            final RemoteSegmentMetadata mdFile = getRemoteSegmentMetadata();
             // During initial recovery flow, the remote store might not
             // have metadata as primary hasn't uploaded anything yet.
             if (mdFile == null && indexShard.state().equals(IndexShardState.STARTED) == false) {
@@ -106,39 +111,50 @@ public void getSegmentFiles(
             }
             logger.debug("Downloading segment files from remote store {}", filesToFetch);
 
-            RemoteSegmentMetadata remoteSegmentMetadata = remoteDirectory.readLatestMetadataFile();
-            Collection directoryFiles = List.of(indexShard.store().directory().listAll());
-            if (remoteSegmentMetadata != null) {
-                try {
-                    indexShard.store().incRef();
-                    indexShard.remoteStore().incRef();
-                    final Directory storeDirectory = indexShard.store().directory();
-                    final List toDownloadSegmentNames = new ArrayList<>();
-                    for (StoreFileMetadata fileMetadata : filesToFetch) {
-                        String file = fileMetadata.name();
-                        assert directoryFiles.contains(file) == false : "Local store already contains the file " + file;
-                        toDownloadSegmentNames.add(file);
-                    }
-                    indexShard.getFileDownloader()
-                        .download(
-                            remoteDirectory,
-                            new ReplicationStatsDirectoryWrapper(storeDirectory, fileProgressTracker),
-                            toDownloadSegmentNames
-                        );
-                    logger.debug("Downloaded segment files from remote store {}", filesToFetch);
-                } finally {
-                    indexShard.store().decRef();
-                    indexShard.remoteStore().decRef();
+            if (remoteMetadataExists()) {
+                final Directory storeDirectory = indexShard.store().directory();
+                final Collection directoryFiles = List.of(storeDirectory.listAll());
+                final List toDownloadSegmentNames = new ArrayList<>();
+                for (StoreFileMetadata fileMetadata : filesToFetch) {
+                    String file = fileMetadata.name();
+                    assert directoryFiles.contains(file) == false : "Local store already contains the file " + file;
+                    toDownloadSegmentNames.add(file);
                 }
+                indexShard.getFileDownloader()
+                    .downloadAsync(
+                        cancellableThreads,
+                        remoteDirectory,
+                        new ReplicationStatsDirectoryWrapper(storeDirectory, fileProgressTracker),
+                        toDownloadSegmentNames,
+                        ActionListener.map(listener, r -> new GetSegmentFilesResponse(filesToFetch))
+                    );
+            } else {
+                listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
             }
-            listener.onResponse(new GetSegmentFilesResponse(filesToFetch));
-        } catch (Exception e) {
+        } catch (IOException | RuntimeException e) {
             listener.onFailure(e);
         }
     }
 
+    @Override
+    public void cancel() {
+        this.cancellableThreads.cancel("Canceled by target");
+    }
+
     @Override
     public String getDescription() {
         return "RemoteStoreReplicationSource";
     }
+
+    private boolean remoteMetadataExists() throws IOException {
+        final AtomicBoolean metadataExists = new AtomicBoolean(false);
+        cancellableThreads.executeIO(() -> metadataExists.set(remoteDirectory.readLatestMetadataFile() != null));
+        return metadataExists.get();
+    }
+
+    private RemoteSegmentMetadata getRemoteSegmentMetadata() throws IOException {
+        AtomicReference mdFile = new AtomicReference<>();
+        cancellableThreads.executeIO(() -> mdFile.set(remoteDirectory.init()));
+        return mdFile.get();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java
index ec6b4d06b32c3..aac59df4f6573 100644
--- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java
+++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTarget.java
@@ -91,6 +91,9 @@ public ReplicationTarget(String name, IndexShard indexShard, ReplicationLuceneIn
         // make sure the store is not released until we are done.
         this.cancellableThreads = new CancellableThreads();
         store.incRef();
+        if (indexShard.indexSettings().isRemoteStoreEnabled()) {
+            indexShard.remoteStore().incRef();
+        }
     }
 
     public long getId() {
@@ -278,6 +281,12 @@ public abstract void writeFileChunk(
     );
 
     protected void closeInternal() {
-        store.decRef();
+        try {
+            store.decRef();
+        } finally {
+            if (indexShard.indexSettings().isRemoteStoreEnabled()) {
+                indexShard.remoteStore().decRef();
+            }
+        }
     }
 }
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java b/server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java
index 588d9e8bb13a2..6d8b3fe4d69fb 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteStoreFileDownloaderTests.java
@@ -9,12 +9,18 @@
 package org.opensearch.index.store;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.NIOFSDirectory;
+import org.opensearch.OpenSearchTimeoutException;
+import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.CancellableThreads;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.test.OpenSearchTestCase;
@@ -31,8 +37,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class RemoteStoreFileDownloaderTests extends OpenSearchTestCase {
 
@@ -76,31 +84,132 @@ public void stopThreadPool() throws Exception {
     }
 
     public void testDownload() throws IOException {
-        fileDownloader.download(source, destination, files.keySet());
+        final PlainActionFuture l = new PlainActionFuture<>();
+        fileDownloader.downloadAsync(new CancellableThreads(), source, destination, files.keySet(), l);
+        l.actionGet();
         assertContent(files, destination);
     }
 
-    public void testDownloadWithSecondDestination() throws IOException {
+    public void testDownloadWithSecondDestination() throws IOException, InterruptedException {
         fileDownloader.download(source, destination, secondDestination, files.keySet(), () -> {});
         assertContent(files, destination);
         assertContent(files, secondDestination);
     }
 
-    public void testDownloadWithFileCompletionHandler() throws IOException {
+    public void testDownloadWithFileCompletionHandler() throws IOException, InterruptedException {
         final AtomicInteger counter = new AtomicInteger(0);
         fileDownloader.download(source, destination, null, files.keySet(), counter::incrementAndGet);
         assertContent(files, destination);
         assertEquals(files.size(), counter.get());
     }
 
-    public void testDownloadNonExistentFile() {
-        assertThrows(NoSuchFileException.class, () -> fileDownloader.download(source, destination, Set.of("not real")));
+    public void testDownloadNonExistentFile() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+        fileDownloader.downloadAsync(new CancellableThreads(), source, destination, Set.of("not real"), new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {}
+
+            @Override
+            public void onFailure(Exception e) {
+                assertEquals(NoSuchFileException.class, e.getClass());
+                latch.countDown();
+            }
+        });
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
     }
 
-    public void testDownloadExtraNonExistentFile() {
-        List filesWithExtra = new ArrayList<>(files.keySet());
+    public void testDownloadExtraNonExistentFile() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final List filesWithExtra = new ArrayList<>(files.keySet());
         filesWithExtra.add("not real");
-        assertThrows(NoSuchFileException.class, () -> fileDownloader.download(source, destination, filesWithExtra));
+        fileDownloader.downloadAsync(new CancellableThreads(), source, destination, filesWithExtra, new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {}
+
+            @Override
+            public void onFailure(Exception e) {
+                assertEquals(NoSuchFileException.class, e.getClass());
+                latch.countDown();
+            }
+        });
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
+    }
+
+    public void testCancellable() {
+        final CancellableThreads cancellableThreads = new CancellableThreads();
+        final PlainActionFuture blockingListener = new PlainActionFuture<>();
+        final Directory blockingDestination = new FilterDirectory(destination) {
+            @Override
+            public void copyFrom(Directory from, String src, String dest, IOContext context) {
+                try {
+                    Thread.sleep(60_000); // Will be interrupted
+                    fail("Expected to be interrupted");
+                } catch (InterruptedException e) {
+                    throw new RuntimeException("Failed due to interrupt", e);
+                }
+            }
+        };
+        fileDownloader.downloadAsync(cancellableThreads, source, blockingDestination, files.keySet(), blockingListener);
+        assertThrows(
+            "Expected to timeout due to blocking directory",
+            OpenSearchTimeoutException.class,
+            () -> blockingListener.actionGet(TimeValue.timeValueMillis(500))
+        );
+        cancellableThreads.cancel("test");
+        assertThrows(
+            "Expected to complete with cancellation failure",
+            CancellableThreads.ExecutionCancelledException.class,
+            blockingListener::actionGet
+        );
+    }
+
+    public void testBlockingCallCanBeInterrupted() throws Exception {
+        final Directory blockingDestination = new FilterDirectory(destination) {
+            @Override
+            public void copyFrom(Directory from, String src, String dest, IOContext context) {
+                try {
+                    Thread.sleep(60_000); // Will be interrupted
+                    fail("Expected to be interrupted");
+                } catch (InterruptedException e) {
+                    throw new RuntimeException("Failed due to interrupt", e);
+                }
+            }
+        };
+        final AtomicReference capturedException = new AtomicReference<>();
+        final Thread thread = new Thread(() -> {
+            try {
+                fileDownloader.download(source, blockingDestination, null, files.keySet(), () -> {});
+            } catch (Exception e) {
+                capturedException.set(e);
+            }
+        });
+        thread.start();
+        thread.interrupt();
+        thread.join();
+        assertEquals(InterruptedException.class, capturedException.get().getClass());
+    }
+
+    public void testIOException() throws IOException, InterruptedException {
+        final Directory failureDirectory = new FilterDirectory(destination) {
+            @Override
+            public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
+                throw new IOException("test");
+            }
+        };
+        assertThrows(IOException.class, () -> fileDownloader.download(source, failureDirectory, null, files.keySet(), () -> {}));
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        fileDownloader.downloadAsync(new CancellableThreads(), source, failureDirectory, files.keySet(), new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {}
+
+            @Override
+            public void onFailure(Exception e) {
+                assertEquals(IOException.class, e.getClass());
+                latch.countDown();
+            }
+        });
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
     }
 
     private static void assertContent(Map expected, Directory destination) throws IOException {
diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
index 1bb1e44a8a600..0ee889af5ce1a 100644
--- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -331,6 +331,12 @@ public static void blockNodeOnAnyFiles(String repository, String nodeName) {
         );
     }
 
+    public static void blockNodeOnAnySegmentFile(String repository, String nodeName) {
+        ((MockRepository) internalCluster().getInstance(RepositoriesService.class, nodeName).repository(repository)).blockOnSegmentFiles(
+            true
+        );
+    }
+
     public static void blockDataNode(String repository, String nodeName) {
         ((MockRepository) internalCluster().getInstance(RepositoriesService.class, nodeName).repository(repository)).blockOnDataFiles(true);
     }
diff --git a/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java
index 7db71c4be0968..72c4ba44d0a31 100644
--- a/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java
+++ b/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java
@@ -139,6 +139,8 @@ public long getFailureCount() {
 
     private volatile boolean blockOnDataFiles;
 
+    private volatile boolean blockOnSegmentFiles;
+
     private volatile boolean blockOnDeleteIndexN;
 
     /**
@@ -190,6 +192,7 @@ public MockRepository(
         maximumNumberOfFailures = metadata.settings().getAsLong("max_failure_number", 100L);
         blockOnAnyFiles = metadata.settings().getAsBoolean("block_on_control", false);
         blockOnDataFiles = metadata.settings().getAsBoolean("block_on_data", false);
+        blockOnSegmentFiles = metadata.settings().getAsBoolean("block_on_segment", false);
         blockAndFailOnWriteSnapFile = metadata.settings().getAsBoolean("block_on_snap", false);
         randomPrefix = metadata.settings().get("random", "default");
         waitAfterUnblock = metadata.settings().getAsLong("wait_after_unblock", 0L);
@@ -237,6 +240,7 @@ public synchronized void unblock() {
         blocked = false;
         // Clean blocking flags, so we wouldn't try to block again
         blockOnDataFiles = false;
+        blockOnSegmentFiles = false;
         blockOnAnyFiles = false;
         blockAndFailOnWriteIndexFile = false;
         blockOnWriteIndexFile = false;
@@ -259,6 +263,14 @@ public void setBlockOnAnyFiles(boolean blocked) {
         blockOnAnyFiles = blocked;
     }
 
+    public void blockOnSegmentFiles(boolean blocked) {
+        blockOnSegmentFiles = blocked;
+    }
+
+    public void setBlockOnSegmentFiles(boolean blocked) {
+        blockOnSegmentFiles = blocked;
+    }
+
     public void setBlockAndFailOnWriteSnapFiles(boolean blocked) {
         blockAndFailOnWriteSnapFile = blocked;
     }
@@ -306,6 +318,7 @@ private synchronized boolean blockExecution() {
         boolean wasBlocked = false;
         try {
             while (blockOnDataFiles
+                || blockOnSegmentFiles
                 || blockOnAnyFiles
                 || blockAndFailOnWriteIndexFile
                 || blockOnWriteIndexFile
@@ -407,6 +420,8 @@ private void maybeIOExceptionOrBlock(String blobName) throws IOException {
                         blockExecutionAndMaybeWait(blobName);
                     } else if (blobName.startsWith("snap-") && blockAndFailOnWriteSnapFile) {
                         blockExecutionAndFail(blobName);
+                    } else if (blockOnSegmentFiles && blobName.contains(".si__")) {
+                        blockExecutionAndMaybeWait(blobName);
                     }
                 }
             }

From c400d84f0e884217454ddfcc1503d02e0b280fa9 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Fri, 20 Oct 2023 17:39:58 +0530
Subject: [PATCH 197/388] [Remote State] fix lock release before deletion is
 completed (#10611)

* fix lock release before deletion is completed

Signed-off-by: bansvaru 
---
 .../remote/RemoteClusterStateService.java     |  8 +++--
 .../RemoteClusterStateServiceTests.java       | 34 +++++++++++++++++++
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index b9d06c8fbb1c1..96ce2fc779ea0 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -1072,7 +1072,8 @@ public void onFailure(Exception e) {
      * @param clusterUUID uuid of cluster state to refer to in remote
      * @param manifestsToRetain no of latest manifest files to keep in remote
      */
-    private void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int manifestsToRetain) {
+    // package private for testing
+    void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int manifestsToRetain) {
         if (deleteStaleMetadataRunning.compareAndSet(false, true) == false) {
             logger.info("Delete stale cluster metadata task is already in progress.");
             return;
@@ -1109,8 +1110,9 @@ public void onFailure(Exception e) {
                     }
                 }
             );
-        } finally {
+        } catch (Exception e) {
             deleteStaleMetadataRunning.set(false);
+            throw e;
         }
     }
 
@@ -1190,7 +1192,7 @@ private void deleteStalePaths(String clusterName, String clusterUUID, List {
             String clusterName = clusterState.getClusterName().value();
-            logger.info("Deleting stale cluster UUIDs data from remote [{}]", clusterName);
+            logger.debug("Deleting stale cluster UUIDs data from remote [{}]", clusterName);
             Set allClustersUUIDsInRemote;
             try {
                 allClustersUUIDsInRemote = new HashSet<>(getAllClusterUUIDs(clusterState.getClusterName().value()));
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 49b7f0ff8d1a9..433eac63e9580 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -62,6 +62,9 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
 import org.mockito.ArgumentCaptor;
@@ -73,6 +76,7 @@
 import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
@@ -1004,6 +1008,36 @@ public void testFileNames() {
         assertThat(splittedName[3], is("P"));
     }
 
+    public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Exception {
+        BlobContainer blobContainer = mock(BlobContainer.class);
+        BlobPath blobPath = new BlobPath().add("random-path");
+        when((blobStoreRepository.basePath())).thenReturn(blobPath);
+        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicInteger callCount = new AtomicInteger(0);
+        doAnswer(invocation -> {
+            callCount.incrementAndGet();
+            if (latch.await(5000, TimeUnit.SECONDS) == false) {
+                throw new Exception("Timed out waiting for delete task queuing to complete");
+            }
+            return null;
+        }).when(blobContainer)
+            .listBlobsByPrefixInSortedOrder(
+                any(String.class),
+                any(int.class),
+                any(BlobContainer.BlobNameSortOrder.class),
+                any(ActionListener.class)
+            );
+
+        remoteClusterStateService.start();
+        remoteClusterStateService.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS);
+        remoteClusterStateService.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS);
+
+        latch.countDown();
+        assertBusy(() -> assertEquals(1, callCount.get()));
+    }
+
     private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException {
         final BlobPath blobPath = mock(BlobPath.class);
         when((blobStoreRepository.basePath())).thenReturn(blobPath);

From 6641ef8a0a39d86aadd272936cae25a406eb3485 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Fri, 20 Oct 2023 18:06:40 +0530
Subject: [PATCH 198/388] Restore remote index shards with
 ExistingStoreRecoverySource after restore from remote state (#10665)

* Restore remote index shards with ExistingStoreRecoverySource after restore from remote state

Signed-off-by: bansvaru 
---
 .../remotestore/BaseRemoteStoreRestoreIT.java |   5 +-
 .../RemoteStoreClusterStateRestoreIT.java     |  28 ++-
 .../cluster/routing/IndexRoutingTable.java    |   4 +-
 .../gateway/ClusterStateUpdaters.java         |  17 +-
 .../recovery/RemoteStoreRestoreService.java   |  30 +--
 .../gateway/ClusterStateUpdatersTests.java    | 237 ++----------------
 6 files changed, 53 insertions(+), 268 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java
index b8481610869e6..99c5d7fb2bae7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java
@@ -46,7 +46,10 @@ protected Collection> nodePlugins() {
     }
 
     protected void restore(String... indices) {
-        boolean restoreAllShards = randomBoolean();
+        restore(randomBoolean(), indices);
+    }
+
+    protected void restore(boolean restoreAllShards, String... indices) {
         if (restoreAllShards) {
             assertAcked(client().admin().indices().prepareClose(indices));
         }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index 3a3e293de9b13..c2cb7cc60f152 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -65,6 +65,13 @@ private void resetCluster(int dataNodeCount, int clusterManagerNodeCount) {
         internalCluster().startDataOnlyNodes(dataNodeCount);
     }
 
+    protected void verifyRedIndicesAndTriggerRestore(Map indexStats, String indexName, boolean indexMoreDocs)
+        throws Exception {
+        ensureRed(indexName);
+        restore(false, indexName);
+        verifyRestoredData(indexStats, indexName, indexMoreDocs);
+    }
+
     public void testFullClusterRestore() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
@@ -83,7 +90,7 @@ public void testFullClusterRestore() throws Exception {
 
         // Step - 3 Trigger full cluster restore and validate
         validateMetadata(List.of(INDEX_NAME));
-        verifyRestoredData(indexStats, INDEX_NAME);
+        verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true);
     }
 
     public void testFullClusterRestoreMultipleIndices() throws Exception {
@@ -112,8 +119,8 @@ public void testFullClusterRestoreMultipleIndices() throws Exception {
 
         // Step - 3 Trigger full cluster restore
         validateMetadata(List.of(INDEX_NAME, secondIndexName));
-        verifyRestoredData(indexStats, INDEX_NAME);
-        verifyRestoredData(indexStats2, secondIndexName, false);
+        verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, false);
+        verifyRedIndicesAndTriggerRestore(indexStats2, secondIndexName, false);
         assertTrue(INDEX_READ_ONLY_SETTING.get(clusterService().state().metadata().index(secondIndexName).getSettings()));
         assertThrows(ClusterBlockException.class, () -> indexSingleDoc(secondIndexName));
         // Test is complete
@@ -181,7 +188,7 @@ public void testRemoteStateFullRestart() throws Exception {
         String newClusterUUID = clusterService().state().metadata().clusterUUID();
         assert Objects.equals(newClusterUUID, prevClusterUUID) : "Full restart not successful. cluster uuid has changed";
         validateCurrentMetadata();
-        verifyRestoredData(indexStats, INDEX_NAME);
+        verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true);
     }
 
     private void validateMetadata(List indexNames) {
@@ -246,19 +253,18 @@ public void testFullClusterRestoreGlobalMetadata() throws Exception {
 
         // Step - 3 Trigger full cluster restore and validate
         // validateCurrentMetadata();
-        verifyRestoredData(indexStats, INDEX_NAME, false);
-
-        // validate global metadata restored
-        verifyRestoredRepositories();
-        verifyRestoredIndexTemplate();
         assertEquals(Integer.valueOf(34), SETTING_CLUSTER_MAX_SHARDS_PER_NODE.get(clusterService().state().metadata().settings()));
         assertEquals(true, SETTING_READ_ONLY_SETTING.get(clusterService().state().metadata().settings()));
         assertTrue(clusterService().state().blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK));
-        // Test is complete
-
         // Remote the cluster read only block to ensure proper cleanup
         updatePersistentSettings(Settings.builder().put(SETTING_READ_ONLY_SETTING.getKey(), false).build());
         assertFalse(clusterService().state().blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK));
+
+        verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, false);
+
+        // validate global metadata restored
+        verifyRestoredRepositories();
+        verifyRestoredIndexTemplate();
     }
 
     private void registerCustomRepository() {
diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
index b12698c8a320e..d77d44580798a 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
@@ -466,12 +466,12 @@ public Builder initializeAsRemoteStoreRestore(
             }
             for (int shardNumber = 0; shardNumber < indexMetadata.getNumberOfShards(); shardNumber++) {
                 ShardId shardId = new ShardId(index, shardNumber);
-                if (forceRecoverAllPrimaries == false && indexShardRoutingTableMap.containsKey(shardId) == false) {
+                if (indexShardRoutingTableMap.containsKey(shardId) == false) {
                     throw new IllegalStateException("IndexShardRoutingTable is not present for shardId: " + shardId);
                 }
                 IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
                 IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingTableMap.get(shardId);
-                if (forceRecoverAllPrimaries || indexShardRoutingTable == null || indexShardRoutingTable.primaryShard().unassigned()) {
+                if (forceRecoverAllPrimaries || indexShardRoutingTable.primaryShard().unassigned()) {
                     // Primary shard to be recovered from remote store.
                     indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, true, recoverySource, unassignedInfo));
                     // All the replica shards to be recovered from peer recovery.
diff --git a/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java
index 4c562b348f141..1563ac84bdd1c 100644
--- a/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java
+++ b/server/src/main/java/org/opensearch/gateway/ClusterStateUpdaters.java
@@ -41,7 +41,6 @@
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
-import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.common.settings.ClusterSettings;
 
@@ -121,21 +120,7 @@ static ClusterState updateRoutingTable(final ClusterState state) {
         // initialize all index routing tables as empty
         final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(state.routingTable());
         for (final IndexMetadata cursor : state.metadata().indices().values()) {
-            // Whether IndexMetadata is recovered from local disk or remote it doesn't matter to us at this point.
-            // We are only concerned about index data recovery here. Which is why we only check for remote store enabled and not for remote
-            // cluster state enabled.
-            if (cursor.getSettings().getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) == false
-                || state.routingTable().hasIndex(cursor.getIndex()) == false
-                || state.routingTable()
-                    .index(cursor.getIndex())
-                    .shardsMatchingPredicateCount(
-                        shardRouting -> shardRouting.primary()
-                            // We need to ensure atleast one of the primaries is being recovered from remote.
-                            // This ensures we have gone through the RemoteStoreRestoreService and routing table is updated
-                            && shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource
-                    ) == 0) {
-                routingTableBuilder.addAsRecovery(cursor);
-            }
+            routingTableBuilder.addAsRecovery(cursor);
         }
         // start with 0 based versions for routing table
         routingTableBuilder.version(0);
diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
index ac9cf35d1d8e5..6692d521b8f65 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
@@ -183,6 +183,7 @@ private RemoteRestoreResult executeRestore(
         final String restoreUUID = UUIDs.randomBase64UUID();
         List indicesToBeRestored = new ArrayList<>();
         int totalShards = 0;
+        boolean metadataFromRemoteStore = false;
         ClusterState.Builder builder = ClusterState.builder(currentState);
         Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
         ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
@@ -190,7 +191,7 @@ private RemoteRestoreResult executeRestore(
         for (Map.Entry> indexMetadataEntry : indexMetadataMap.entrySet()) {
             String indexName = indexMetadataEntry.getKey();
             IndexMetadata indexMetadata = indexMetadataEntry.getValue().v2();
-            boolean metadataFromRemoteStore = indexMetadataEntry.getValue().v1();
+            metadataFromRemoteStore = indexMetadataEntry.getValue().v1();
             IndexMetadata updatedIndexMetadata = indexMetadata;
             if (metadataFromRemoteStore == false && restoreAllShards) {
                 updatedIndexMetadata = IndexMetadata.builder(indexMetadata)
@@ -204,27 +205,23 @@ private RemoteRestoreResult executeRestore(
 
             IndexId indexId = new IndexId(indexName, updatedIndexMetadata.getIndexUUID());
 
-            Map indexShardRoutingTableMap = new HashMap<>();
             if (metadataFromRemoteStore == false) {
-                indexShardRoutingTableMap = currentState.routingTable()
+                Map indexShardRoutingTableMap = currentState.routingTable()
                     .index(indexName)
                     .shards()
                     .values()
                     .stream()
                     .collect(Collectors.toMap(IndexShardRoutingTable::shardId, Function.identity()));
+
+                RecoverySource.RemoteStoreRecoverySource recoverySource = new RecoverySource.RemoteStoreRecoverySource(
+                    restoreUUID,
+                    updatedIndexMetadata.getCreationVersion(),
+                    indexId
+                );
+
+                rtBuilder.addAsRemoteStoreRestore(updatedIndexMetadata, recoverySource, indexShardRoutingTableMap, restoreAllShards);
             }
 
-            RecoverySource.RemoteStoreRecoverySource recoverySource = new RecoverySource.RemoteStoreRecoverySource(
-                restoreUUID,
-                updatedIndexMetadata.getCreationVersion(),
-                indexId
-            );
-            rtBuilder.addAsRemoteStoreRestore(
-                updatedIndexMetadata,
-                recoverySource,
-                indexShardRoutingTableMap,
-                restoreAllShards || metadataFromRemoteStore
-            );
             blocks.updateBlocks(updatedIndexMetadata);
             mdBuilder.put(updatedIndexMetadata, true);
             indicesToBeRestored.add(indexName);
@@ -239,7 +236,10 @@ private RemoteRestoreResult executeRestore(
 
         RoutingTable rt = rtBuilder.build();
         ClusterState updatedState = builder.metadata(mdBuilder).blocks(blocks).routingTable(rt).build();
-        return RemoteRestoreResult.build(restoreUUID, restoreInfo, allocationService.reroute(updatedState, "restored from remote store"));
+        if (metadataFromRemoteStore == false) {
+            updatedState = allocationService.reroute(updatedState, "restored from remote store");
+        }
+        return RemoteRestoreResult.build(restoreUUID, restoreInfo, updatedState);
     }
 
     private void restoreGlobalMetadata(Metadata.Builder mdBuilder, Metadata remoteMetadata) {
diff --git a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java
index 9b3fd45245ef7..1c43bb565ef69 100644
--- a/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java
+++ b/server/src/test/java/org/opensearch/gateway/ClusterStateUpdatersTests.java
@@ -41,6 +41,7 @@
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.routing.IndexRoutingTable;
+import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.cluster.routing.UnassignedInfo;
@@ -52,12 +53,14 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.index.Index;
+import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.repositories.IndexId;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
@@ -275,7 +278,7 @@ public void testUpdateRoutingTable() {
         }
     }
 
-    public void testSkipRoutingTableUpdateWhenRemoteRecovery() {
+    public void testRoutingTableUpdateWhenRemoteStateRecovery() {
         final int numOfShards = randomIntBetween(1, 10);
 
         final IndexMetadata remoteMetadata = createIndexMetadata(
@@ -286,7 +289,7 @@ public void testSkipRoutingTableUpdateWhenRemoteRecovery() {
                 .build()
         );
 
-        // Test remote index routing table is generated with ExistingStoreRecoverySource if no routing table is present
+        // Test remote index routing table is generated with ExistingStoreRecoverySource
         {
             final Index index = remoteMetadata.getIndex();
             final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE)
@@ -322,48 +325,14 @@ public void testSkipRoutingTableUpdateWhenRemoteRecovery() {
 
         }
 
-        // Test remote index routing table is overridden if recovery source is not RemoteStoreRecoverySource
+        // Test remote index routing table is overridden if recovery source is RemoteStoreRecoverySource
         {
-            IndexRoutingTable.Builder remoteBuilderWithoutRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex())
-                .initializeAsNew(remoteMetadata);
             final Index index = remoteMetadata.getIndex();
-            final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE)
-                .metadata(Metadata.builder().put(remoteMetadata, false).build())
-                .routingTable(new RoutingTable.Builder().add(remoteBuilderWithoutRemoteRecovery.build()).build())
-                .build();
-            assertTrue(initialState.routingTable().hasIndex(index));
-            final ClusterState newState = updateRoutingTable(initialState);
-            IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex());
-            assertTrue(newState.routingTable().hasIndex(index));
-            assertEquals(
-                0,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.INDEX_CREATED)
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED)
-                )
-            );
-            assertEquals(
-                0,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource
-                )
-            );
-
-        }
-
-        // Test routing table update is skipped for a remote index
-        {
+            Map routingTableMap = new HashMap<>();
+            for (int shardNumber = 0; shardNumber < remoteMetadata.getNumberOfShards(); shardNumber++) {
+                ShardId shardId = new ShardId(index, shardNumber);
+                routingTableMap.put(shardId, new IndexShardRoutingTable.Builder(new ShardId(remoteMetadata.getIndex(), 1)).build());
+            }
             IndexRoutingTable.Builder remoteBuilderWithRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex())
                 .initializeAsRemoteStoreRestore(
                     remoteMetadata,
@@ -372,10 +341,9 @@ public void testSkipRoutingTableUpdateWhenRemoteRecovery() {
                         remoteMetadata.getCreationVersion(),
                         new IndexId(remoteMetadata.getIndex().getName(), remoteMetadata.getIndexUUID())
                     ),
-                    new HashMap<>(),
+                    routingTableMap,
                     true
                 );
-            final Index index = remoteMetadata.getIndex();
             final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE)
                 .metadata(Metadata.builder().put(remoteMetadata, false).build())
                 .routingTable(new RoutingTable.Builder().add(remoteBuilderWithRemoteRecovery.build()).build())
@@ -387,205 +355,28 @@ public void testSkipRoutingTableUpdateWhenRemoteRecovery() {
             assertEquals(
                 0,
                 newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED)
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED)
-                )
-            );
-            assertEquals(
-                0,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource
-                )
-            );
-
-        }
-
-        // Test reset routing table for 2 indices - one remote and one non remote.
-        // Routing table for non remote index should be updated and remote index routing table should remain intact
-        {
-            final IndexMetadata nonRemoteMetadata = createIndexMetadata(
-                "test-nonremote",
-                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards).build()
-            );
-            IndexRoutingTable.Builder remoteBuilderWithRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex())
-                .initializeAsRemoteStoreRestore(
-                    remoteMetadata,
-                    new RecoverySource.RemoteStoreRecoverySource(
-                        UUIDs.randomBase64UUID(),
-                        remoteMetadata.getCreationVersion(),
-                        new IndexId(remoteMetadata.getIndex().getName(), remoteMetadata.getIndexUUID())
-                    ),
-                    new HashMap<>(),
-                    true
-                );
-            IndexRoutingTable.Builder nonRemoteBuilderWithoutRemoteRecovery = new IndexRoutingTable.Builder(nonRemoteMetadata.getIndex())
-                .initializeAsNew(nonRemoteMetadata);
-            final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE)
-                .metadata(Metadata.builder().put(remoteMetadata, false).build())
-                .metadata(Metadata.builder().put(nonRemoteMetadata, false).build())
-                .routingTable(
-                    new RoutingTable.Builder().add(remoteBuilderWithRemoteRecovery.build())
-                        .add(nonRemoteBuilderWithoutRemoteRecovery.build())
-                        .build()
-                )
-                .build();
-            assertTrue(initialState.routingTable().hasIndex(remoteMetadata.getIndex()));
-            assertTrue(initialState.routingTable().hasIndex(nonRemoteMetadata.getIndex()));
-            final ClusterState newState = updateRoutingTable(initialState);
-            assertTrue(newState.routingTable().hasIndex(remoteMetadata.getIndex()));
-            assertTrue(newState.routingTable().hasIndex(nonRemoteMetadata.getIndex()));
-            IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex());
-            IndexRoutingTable newNonRemoteIndexRoutingTable = newState.routingTable().index(nonRemoteMetadata.getIndex());
-            assertEquals(
-                0,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED)
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED)
-                )
-            );
-            assertEquals(
-                0,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource
-                )
-            );
-            assertEquals(
-                0,
-                newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount(
                     shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.INDEX_CREATED)
                 )
             );
             assertEquals(
                 numOfShards,
-                newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED)
-                )
-            );
-            assertEquals(
-                0,
-                newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newNonRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource
-                )
-            );
-        }
-
-        // Test reset routing table for 2 indices, both remote backed but only once index has RemoteStoreRecoverySource.
-        // Routing table for only remote index without RemoteStoreRecoverySource should be updated
-        {
-            final IndexMetadata remoteWithoutRemoteRecoveryMetadata = createIndexMetadata(
-                "test-remote-without-recovery",
-                Settings.builder()
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
-                    .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
-                    .build()
-            );
-            IndexRoutingTable.Builder remoteBuilderWithRemoteRecovery = new IndexRoutingTable.Builder(remoteMetadata.getIndex())
-                .initializeAsRemoteStoreRestore(
-                    remoteMetadata,
-                    new RecoverySource.RemoteStoreRecoverySource(
-                        UUIDs.randomBase64UUID(),
-                        remoteMetadata.getCreationVersion(),
-                        new IndexId(remoteMetadata.getIndex().getName(), remoteMetadata.getIndexUUID())
-                    ),
-                    new HashMap<>(),
-                    true
-                );
-            IndexRoutingTable.Builder remoteBuilderWithoutRemoteRecovery = new IndexRoutingTable.Builder(
-                remoteWithoutRemoteRecoveryMetadata.getIndex()
-            ).initializeAsNew(remoteWithoutRemoteRecoveryMetadata);
-            final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE)
-                .metadata(Metadata.builder().put(remoteMetadata, false).build())
-                .metadata(Metadata.builder().put(remoteWithoutRemoteRecoveryMetadata, false).build())
-                .routingTable(
-                    new RoutingTable.Builder().add(remoteBuilderWithRemoteRecovery.build())
-                        .add(remoteBuilderWithoutRemoteRecovery.build())
-                        .build()
-                )
-                .build();
-            assertTrue(initialState.routingTable().hasIndex(remoteMetadata.getIndex()));
-            assertTrue(initialState.routingTable().hasIndex(remoteWithoutRemoteRecoveryMetadata.getIndex()));
-            final ClusterState newState = updateRoutingTable(initialState);
-            assertTrue(newState.routingTable().hasIndex(remoteMetadata.getIndex()));
-            assertTrue(newState.routingTable().hasIndex(remoteWithoutRemoteRecoveryMetadata.getIndex()));
-            IndexRoutingTable newRemoteIndexRoutingTable = newState.routingTable().index(remoteMetadata.getIndex());
-            IndexRoutingTable newRemoteWithoutRemoteRecoveryIndexRoutingTable = newState.routingTable()
-                .index(remoteWithoutRemoteRecoveryMetadata.getIndex());
-            assertEquals(
-                0,
                 newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
                     shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED)
                 )
             );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED)
-                )
-            );
             assertEquals(
                 0,
                 newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource
-                )
-            );
-            assertEquals(
-                0,
-                newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.INDEX_CREATED)
-                )
-            );
-            assertEquals(
-                numOfShards,
-                newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount(
-                    shardRouting -> shardRouting.unassignedInfo().getReason().equals(UnassignedInfo.Reason.CLUSTER_RECOVERED)
-                )
-            );
-            assertEquals(
-                0,
-                newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount(
                     shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.RemoteStoreRecoverySource
                 )
             );
             assertEquals(
                 numOfShards,
-                newRemoteWithoutRemoteRecoveryIndexRoutingTable.shardsMatchingPredicateCount(
+                newRemoteIndexRoutingTable.shardsMatchingPredicateCount(
                     shardRouting -> shardRouting.recoverySource() instanceof RecoverySource.EmptyStoreRecoverySource
                 )
             );
+
         }
     }
 

From e12ab0f9ed5cfa32fbbcf654253180a72e50d0d7 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Fri, 20 Oct 2023 18:18:36 +0530
Subject: [PATCH 199/388] Fix flaky remote cluster state UT (#10780)

Signed-off-by: Dhwanil Patel 
---
 .../gateway/remote/RemoteClusterStateServiceTests.java       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 433eac63e9580..bcc58789dd6fd 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -458,7 +458,7 @@ public void testGlobalMetadataOnlyUpdated() throws IOException {
         mockBlobStoreObjects();
         final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build();
         final ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata))
+            .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata).version(randomNonNegativeLong()))
             .build();
         final ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder()
             .codecVersion(2)
@@ -479,6 +479,7 @@ public void testGlobalMetadataOnlyUpdated() throws IOException {
         // new cluster state where only global metadata is different
         Metadata newMetadata = Metadata.builder(clusterState.metadata())
             .persistentSettings(Settings.builder().put("cluster.blocks.read_only", true).build())
+            .version(randomNonNegativeLong())
             .build();
         ClusterState newClusterState = ClusterState.builder(clusterState).metadata(newMetadata).build();
 
@@ -1281,7 +1282,7 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() {
             .version(1L)
             .stateUUID("state-uuid")
             .metadata(
-                Metadata.builder().put(indexMetadata, true).clusterUUID("cluster-uuid").coordinationMetadata(coordinationMetadata).build()
+                Metadata.builder().version(randomNonNegativeLong()).put(indexMetadata, true).clusterUUID("cluster-uuid").coordinationMetadata(coordinationMetadata).build()
             );
     }
 

From 5093cc71476f5796f3b91fb18ff867cfe37588c0 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Fri, 20 Oct 2023 19:18:25 +0530
Subject: [PATCH 200/388] Fix spotless failure (#10782)

Signed-off-by: Dhwanil Patel 
---
 .../gateway/remote/RemoteClusterStateServiceTests.java     | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index bcc58789dd6fd..5202f31c514ed 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -1282,7 +1282,12 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() {
             .version(1L)
             .stateUUID("state-uuid")
             .metadata(
-                Metadata.builder().version(randomNonNegativeLong()).put(indexMetadata, true).clusterUUID("cluster-uuid").coordinationMetadata(coordinationMetadata).build()
+                Metadata.builder()
+                    .version(randomNonNegativeLong())
+                    .put(indexMetadata, true)
+                    .clusterUUID("cluster-uuid")
+                    .coordinationMetadata(coordinationMetadata)
+                    .build()
             );
     }
 

From e691df09c66dcc1693897543fd7633c4b208ce48 Mon Sep 17 00:00:00 2001
From: rayshrey <121871912+rayshrey@users.noreply.github.com>
Date: Fri, 20 Oct 2023 20:36:20 +0530
Subject: [PATCH 201/388] Add tracing instrumentation for indexing paths
 (#10273)

* Add tracing instrumentation for indexing paths

Signed-off-by: Shreyansh Ray 

* Fix failing tests and review changes

Signed-off-by: Shreyansh Ray 

* Fix test failures due to Span not being properly closed

Signed-off-by: Shreyansh Ray 

* Changes to spans in primary and replica actions

Signed-off-by: Shreyansh Ray 

* Review comments fixes and refactoring

Signed-off-by: Shreyansh Ray 

* Precommit auto-changes

Signed-off-by: Shreyansh Ray 

* Add refresh policy as attribute

Signed-off-by: Shreyansh Ray 

* Fix changelog entry

Signed-off-by: Shreyansh Ray 

* Instrument primary/replica write in TransportWriteAction instead of TransportShardBulkAction

Signed-off-by: Shreyansh Ray 

* Modify SpanBuilder

Signed-off-by: Shreyansh Ray 

* spotlessApply and precommit

Signed-off-by: Shreyansh Ray 

* Change span names

Signed-off-by: Shreyansh Ray 

* Pass Noop Tracer instead of injected tracer

Signed-off-by: Shreyansh Ray 

* Reverting previous changes

Signed-off-by: Shreyansh Ray 

* Remove tracer variable from TransportShardBulkAction

Signed-off-by: Shreyansh Ray 

---------

Signed-off-by: Shreyansh Ray 
---
 CHANGELOG.md                                  |   1 +
 .../action/bulk/TransportBulkAction.java      | 110 +++++++++++-------
 .../action/bulk/TransportShardBulkAction.java |   7 +-
 .../TransportResyncReplicationAction.java     |   7 +-
 .../replication/TransportWriteAction.java     |  24 +++-
 .../index/seqno/RetentionLeaseSyncAction.java |   7 +-
 .../telemetry/tracing/AttributeNames.java     |  25 ++++
 .../telemetry/tracing/SpanBuilder.java        |  20 ++++
 ...ActionIndicesThatCannotBeCreatedTests.java |   4 +-
 .../bulk/TransportBulkActionIngestTests.java  |   4 +-
 .../action/bulk/TransportBulkActionTests.java |   3 +-
 .../bulk/TransportBulkActionTookTests.java    |   3 +-
 .../bulk/TransportShardBulkActionTests.java   |  13 ++-
 ...TransportResyncReplicationActionTests.java |   6 +-
 ...rtWriteActionForIndexingPressureTests.java |   3 +-
 .../TransportWriteActionTests.java            |   6 +-
 .../seqno/RetentionLeaseSyncActionTests.java  |  12 +-
 .../snapshots/SnapshotResiliencyTests.java    |   9 +-
 18 files changed, 192 insertions(+), 72 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5c52c43a35b8d..d7d492679c79d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
 - [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
+- Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
 
 ### Deprecated
 
diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java
index 726ba7ba119af..4a9b07c12821d 100644
--- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java
@@ -85,6 +85,11 @@
 import org.opensearch.ingest.IngestService;
 import org.opensearch.node.NodeClosedException;
 import org.opensearch.tasks.Task;
+import org.opensearch.telemetry.tracing.Span;
+import org.opensearch.telemetry.tracing.SpanBuilder;
+import org.opensearch.telemetry.tracing.SpanScope;
+import org.opensearch.telemetry.tracing.Tracer;
+import org.opensearch.telemetry.tracing.listener.TraceableActionListener;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.threadpool.ThreadPool.Names;
 import org.opensearch.transport.TransportService;
@@ -133,6 +138,7 @@ public class TransportBulkAction extends HandledTransportAction() {
-                    @Override
-                    public void onResponse(BulkShardResponse bulkShardResponse) {
-                        for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
-                            // we may have no response if item failed
-                            if (bulkItemResponse.getResponse() != null) {
-                                bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
-                            }
 
-                            docStatusStats.inc(bulkItemResponse.status());
-                            responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
-                        }
+                final Span span = tracer.startSpan(SpanBuilder.from("bulkShardAction", nodeId, bulkShardRequest));
+                try (SpanScope spanScope = tracer.withSpanInScope(span)) {
+                    shardBulkAction.execute(
+                        bulkShardRequest,
+                        TraceableActionListener.create(ActionListener.runBefore(new ActionListener() {
+                            @Override
+                            public void onResponse(BulkShardResponse bulkShardResponse) {
+                                for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
+                                    // we may have no response if item failed
+                                    if (bulkItemResponse.getResponse() != null) {
+                                        bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
+                                    }
 
-                        if (counter.decrementAndGet() == 0) {
-                            finishHim();
-                        }
-                    }
+                                    docStatusStats.inc(bulkItemResponse.status());
+                                    responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
+                                }
 
-                    @Override
-                    public void onFailure(Exception e) {
-                        // create failures for all relevant requests
-                        for (BulkItemRequest request : requests) {
-                            final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
-                            final DocWriteRequest docWriteRequest = request.request();
-                            final BulkItemResponse bulkItemResponse = new BulkItemResponse(
-                                request.id(),
-                                docWriteRequest.opType(),
-                                new BulkItemResponse.Failure(indexName, docWriteRequest.id(), e)
-                            );
+                                if (counter.decrementAndGet() == 0) {
+                                    finishHim();
+                                }
+                            }
 
-                            docStatusStats.inc(bulkItemResponse.status());
-                            responses.set(request.id(), bulkItemResponse);
-                        }
+                            @Override
+                            public void onFailure(Exception e) {
+                                // create failures for all relevant requests
+                                for (BulkItemRequest request : requests) {
+                                    final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
+                                    final DocWriteRequest docWriteRequest = request.request();
+                                    final BulkItemResponse bulkItemResponse = new BulkItemResponse(
+                                        request.id(),
+                                        docWriteRequest.opType(),
+                                        new BulkItemResponse.Failure(indexName, docWriteRequest.id(), e)
+                                    );
+
+                                    docStatusStats.inc(bulkItemResponse.status());
+                                    responses.set(request.id(), bulkItemResponse);
+                                }
 
-                        if (counter.decrementAndGet() == 0) {
-                            finishHim();
-                        }
-                    }
+                                if (counter.decrementAndGet() == 0) {
+                                    finishHim();
+                                }
+                            }
 
-                    private void finishHim() {
-                        indicesService.addDocStatusStats(docStatusStats);
-                        listener.onResponse(
-                            new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos))
-                        );
-                    }
-                }, releasable::close));
+                            private void finishHim() {
+                                indicesService.addDocStatusStats(docStatusStats);
+                                listener.onResponse(
+                                    new BulkResponse(
+                                        responses.toArray(new BulkItemResponse[responses.length()]),
+                                        buildTookInMillis(startTimeNanos)
+                                    )
+                                );
+                            }
+                        }, releasable::close), span, tracer)
+                    );
+                } catch (Exception e) {
+                    span.setError(e);
+                    span.endSpan();
+                    throw e;
+                }
             }
             bulkRequest = null; // allow memory for bulk request items to be reclaimed before all items have been completed
         }
diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
index fddda0ef1f9a7..268a6ed6f85b8 100644
--- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
@@ -99,6 +99,7 @@
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.node.NodeClosedException;
 import org.opensearch.tasks.Task;
+import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.threadpool.ThreadPool.Names;
 import org.opensearch.transport.TransportChannel;
@@ -161,7 +162,8 @@ public TransportShardBulkAction(
         IndexingPressureService indexingPressureService,
         SegmentReplicationPressureService segmentReplicationPressureService,
         RemoteStorePressureService remoteStorePressureService,
-        SystemIndices systemIndices
+        SystemIndices systemIndices,
+        Tracer tracer
     ) {
         super(
             settings,
@@ -177,7 +179,8 @@ public TransportShardBulkAction(
             EXECUTOR_NAME_FUNCTION,
             false,
             indexingPressureService,
-            systemIndices
+            systemIndices,
+            tracer
         );
         this.updateHelper = updateHelper;
         this.mappingUpdatedAction = mappingUpdatedAction;
diff --git a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java
index 032fe83e2220b..9d60706d1f100 100644
--- a/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java
+++ b/server/src/main/java/org/opensearch/action/resync/TransportResyncReplicationAction.java
@@ -54,6 +54,7 @@
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.tasks.Task;
+import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.threadpool.ThreadPool.Names;
 import org.opensearch.transport.TransportException;
@@ -93,7 +94,8 @@ public TransportResyncReplicationAction(
         ShardStateAction shardStateAction,
         ActionFilters actionFilters,
         IndexingPressureService indexingPressureService,
-        SystemIndices systemIndices
+        SystemIndices systemIndices,
+        Tracer tracer
     ) {
         super(
             settings,
@@ -109,7 +111,8 @@ public TransportResyncReplicationAction(
             EXECUTOR_NAME_FUNCTION,
             true, /* we should never reject resync because of thread pool capacity on primary */
             indexingPressureService,
-            systemIndices
+            systemIndices,
+            tracer
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java
index a0b5299805868..9ebfa8cfd0df8 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java
@@ -59,6 +59,11 @@
 import org.opensearch.index.translog.Translog.Location;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.SystemIndices;
+import org.opensearch.telemetry.tracing.Span;
+import org.opensearch.telemetry.tracing.SpanBuilder;
+import org.opensearch.telemetry.tracing.SpanScope;
+import org.opensearch.telemetry.tracing.Tracer;
+import org.opensearch.telemetry.tracing.listener.TraceableActionListener;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
@@ -82,6 +87,7 @@ public abstract class TransportWriteAction<
     protected final SystemIndices systemIndices;
 
     private final Function executorFunction;
+    private final Tracer tracer;
 
     protected TransportWriteAction(
         Settings settings,
@@ -97,7 +103,8 @@ protected TransportWriteAction(
         Function executorFunction,
         boolean forceExecutionOnPrimary,
         IndexingPressureService indexingPressureService,
-        SystemIndices systemIndices
+        SystemIndices systemIndices,
+        Tracer tracer
     ) {
         // We pass ThreadPool.Names.SAME to the super class as we control the dispatching to the
         // ThreadPool.Names.WRITE/ThreadPool.Names.SYSTEM_WRITE thread pools in this class.
@@ -119,6 +126,7 @@ protected TransportWriteAction(
         this.executorFunction = executorFunction;
         this.indexingPressureService = indexingPressureService;
         this.systemIndices = systemIndices;
+        this.tracer = tracer;
     }
 
     protected String executor(IndexShard shard) {
@@ -220,7 +228,12 @@ protected void shardOperationOnPrimary(
         threadPool.executor(executor).execute(new ActionRunnable>(listener) {
             @Override
             protected void doRun() {
-                dispatchedShardOperationOnPrimary(request, primary, listener);
+                Span span = tracer.startSpan(
+                    SpanBuilder.from("dispatchedShardOperationOnPrimary", clusterService.localNode().getId(), request)
+                );
+                try (SpanScope spanScope = tracer.withSpanInScope(span)) {
+                    dispatchedShardOperationOnPrimary(request, primary, TraceableActionListener.create(listener, span, tracer));
+                }
             }
 
             @Override
@@ -248,7 +261,12 @@ protected void shardOperationOnReplica(ReplicaRequest request, IndexShard replic
         threadPool.executor(executorFunction.apply(replica)).execute(new ActionRunnable(listener) {
             @Override
             protected void doRun() {
-                dispatchedShardOperationOnReplica(request, replica, listener);
+                Span span = tracer.startSpan(
+                    SpanBuilder.from("dispatchedShardOperationOnReplica", clusterService.localNode().getId(), request)
+                );
+                try (SpanScope spanScope = tracer.withSpanInScope(span)) {
+                    dispatchedShardOperationOnReplica(request, replica, TraceableActionListener.create(listener, span, tracer));
+                }
             }
 
             @Override
diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java
index f74fc7eefe65c..ca3c7e1d49700 100644
--- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java
+++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseSyncAction.java
@@ -62,6 +62,7 @@
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.tasks.Task;
+import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportException;
 import org.opensearch.transport.TransportResponseHandler;
@@ -99,7 +100,8 @@ public RetentionLeaseSyncAction(
         final ShardStateAction shardStateAction,
         final ActionFilters actionFilters,
         final IndexingPressureService indexingPressureService,
-        final SystemIndices systemIndices
+        final SystemIndices systemIndices,
+        final Tracer tracer
     ) {
         super(
             settings,
@@ -115,7 +117,8 @@ public RetentionLeaseSyncAction(
             ignore -> ThreadPool.Names.MANAGEMENT,
             false,
             indexingPressureService,
-            systemIndices
+            systemIndices,
+            tracer
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java b/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java
index a9514c298ef88..b6b2cf360d1c5 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java
@@ -69,4 +69,29 @@ private AttributeNames() {
      * Action Name.
      */
     public static final String TRANSPORT_ACTION = "action";
+
+    /**
+     * Index Name
+     */
+    public static final String INDEX = "index";
+
+    /**
+     * Shard ID
+     */
+    public static final String SHARD_ID = "shard_id";
+
+    /**
+     * Number of request items in bulk request
+     */
+    public static final String BULK_REQUEST_ITEMS = "bulk_request_items";
+
+    /**
+     * Node ID
+     */
+    public static final String NODE_ID = "node_id";
+
+    /**
+     * Refresh Policy
+     */
+    public static final String REFRESH_POLICY = "refresh_policy";
 }
diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java
index d97fbd371ab2a..1dce422943b7a 100644
--- a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java
+++ b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.telemetry.tracing;
 
+import org.opensearch.action.bulk.BulkShardRequest;
+import org.opensearch.action.support.replication.ReplicatedWriteRequest;
 import org.opensearch.common.annotation.InternalApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.http.HttpRequest;
@@ -68,6 +70,10 @@ public static SpanCreationContext from(String action, Transport.Connection conne
         return SpanCreationContext.server().name(createSpanName(action, connection)).attributes(buildSpanAttributes(action, connection));
     }
 
+    public static SpanCreationContext from(String spanName, String nodeId, ReplicatedWriteRequest request) {
+        return SpanCreationContext.server().name(spanName).attributes(buildSpanAttributes(nodeId, request));
+    }
+
     private static String createSpanName(HttpRequest httpRequest) {
         return httpRequest.method().name() + SEPARATOR + httpRequest.uri();
     }
@@ -150,4 +156,18 @@ private static Attributes buildSpanAttributes(String action, TcpChannel tcpChann
         return attributes;
     }
 
+    private static Attributes buildSpanAttributes(String nodeId, ReplicatedWriteRequest request) {
+        Attributes attributes = Attributes.create()
+            .addAttribute(AttributeNames.NODE_ID, nodeId)
+            .addAttribute(AttributeNames.REFRESH_POLICY, request.getRefreshPolicy().getValue());
+        if (request.shardId() != null) {
+            attributes.addAttribute(AttributeNames.INDEX, request.shardId().getIndexName())
+                .addAttribute(AttributeNames.SHARD_ID, request.shardId().getId());
+        }
+        if (request instanceof BulkShardRequest) {
+            attributes.addAttribute(AttributeNames.BULK_REQUEST_ITEMS, ((BulkShardRequest) request).items().length);
+        }
+        return attributes;
+    }
+
 }
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
index 0f67eff26cbde..cf7080ab2fc06 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
@@ -54,6 +54,7 @@
 import org.opensearch.index.VersionType;
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.tasks.Task;
+import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
 import org.opensearch.threadpool.ThreadPool;
@@ -155,7 +156,8 @@ private void indicesThatCannotBeCreatedTestCase(
                 new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null)
             ),
             null,
-            new SystemIndices(emptyMap())
+            new SystemIndices(emptyMap()),
+            NoopTracer.INSTANCE
         ) {
             @Override
             void executeBulk(
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
index 515f6eae28a34..141c630b94020 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
@@ -70,6 +70,7 @@
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.ingest.IngestService;
 import org.opensearch.tasks.Task;
+import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
 import org.opensearch.threadpool.ThreadPool;
@@ -172,7 +173,8 @@ class TestTransportBulkAction extends TransportBulkAction {
                     new ClusterService(SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null)
                 ),
                 null,
-                new SystemIndices(emptyMap())
+                new SystemIndices(emptyMap()),
+                NoopTracer.INSTANCE
             );
         }
 
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java
index 10cad6fb147a2..6bbd740df7f9c 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTests.java
@@ -118,7 +118,8 @@ class TestTransportBulkAction extends TransportBulkAction {
                 new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(), new SystemIndices(emptyMap())),
                 new IndexingPressureService(Settings.EMPTY, clusterService),
                 mock(IndicesService.class),
-                new SystemIndices(emptyMap())
+                new SystemIndices(emptyMap()),
+                NoopTracer.INSTANCE
             );
         }
 
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java
index 852e3837e1e7a..9d5b4430ea395 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionTookTests.java
@@ -282,7 +282,8 @@ static class TestTransportBulkAction extends TransportBulkAction {
                 new IndexingPressureService(Settings.EMPTY, clusterService),
                 null,
                 new SystemIndices(emptyMap()),
-                relativeTimeProvider
+                relativeTimeProvider,
+                NoopTracer.INSTANCE
             );
         }
 
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java
index fe0fdd07025d9..b325cfa197933 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java
@@ -88,6 +88,7 @@
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.SystemIndices;
+import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.threadpool.ThreadPool.Names;
@@ -1074,7 +1075,8 @@ public void testHandlePrimaryTermValidationRequestWithDifferentAllocationId() {
             mock(IndexingPressureService.class),
             mock(SegmentReplicationPressureService.class),
             mock(RemoteStorePressureService.class),
-            mock(SystemIndices.class)
+            mock(SystemIndices.class),
+            NoopTracer.INSTANCE
         );
         action.handlePrimaryTermValidationRequest(
             new TransportShardBulkAction.PrimaryTermValidationRequest(aId + "-1", 1, shardId),
@@ -1105,7 +1107,8 @@ public void testHandlePrimaryTermValidationRequestWithOlderPrimaryTerm() {
             mock(IndexingPressureService.class),
             mock(SegmentReplicationPressureService.class),
             mock(RemoteStorePressureService.class),
-            mock(SystemIndices.class)
+            mock(SystemIndices.class),
+            NoopTracer.INSTANCE
         );
         action.handlePrimaryTermValidationRequest(
             new TransportShardBulkAction.PrimaryTermValidationRequest(aId, 1, shardId),
@@ -1136,7 +1139,8 @@ public void testHandlePrimaryTermValidationRequestSuccess() {
             mock(IndexingPressureService.class),
             mock(SegmentReplicationPressureService.class),
             mock(RemoteStorePressureService.class),
-            mock(SystemIndices.class)
+            mock(SystemIndices.class),
+            NoopTracer.INSTANCE
         );
         action.handlePrimaryTermValidationRequest(
             new TransportShardBulkAction.PrimaryTermValidationRequest(aId, 1, shardId),
@@ -1178,7 +1182,8 @@ private TransportShardBulkAction createAction() {
             mock(IndexingPressureService.class),
             mock(SegmentReplicationPressureService.class),
             mock(RemoteStorePressureService.class),
-            mock(SystemIndices.class)
+            mock(SystemIndices.class),
+            NoopTracer.INSTANCE
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
index 3bd8930064563..da87a0a967f53 100644
--- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
+++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
@@ -203,7 +203,8 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception {
                     shardStateAction,
                     new ActionFilters(new HashSet<>()),
                     new IndexingPressureService(Settings.EMPTY, clusterService),
-                    new SystemIndices(emptyMap())
+                    new SystemIndices(emptyMap()),
+                    NoopTracer.INSTANCE
                 );
 
                 assertThat(action.globalBlockLevel(), nullValue());
@@ -256,7 +257,8 @@ private TransportResyncReplicationAction createAction() {
             mock(ShardStateAction.class),
             new ActionFilters(new HashSet<>()),
             mock(IndexingPressureService.class),
-            new SystemIndices(emptyMap())
+            new SystemIndices(emptyMap()),
+            NoopTracer.INSTANCE
         );
     }
 }
diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java
index 4a2185d1558f7..7212b1f5efe13 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionForIndexingPressureTests.java
@@ -392,7 +392,8 @@ protected TestAction(
                 ignore -> ThreadPool.Names.SAME,
                 false,
                 TransportWriteActionForIndexingPressureTests.this.indexingPressureService,
-                new SystemIndices(emptyMap())
+                new SystemIndices(emptyMap()),
+                NoopTracer.INSTANCE
             );
         }
 
diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java
index 9d2069ac16190..b4549f82230bf 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java
@@ -477,7 +477,8 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF
                 ignore -> ThreadPool.Names.SAME,
                 false,
                 new IndexingPressureService(Settings.EMPTY, TransportWriteActionTests.this.clusterService),
-                new SystemIndices(emptyMap())
+                new SystemIndices(emptyMap()),
+                NoopTracer.INSTANCE
             );
             this.withDocumentFailureOnPrimary = withDocumentFailureOnPrimary;
             this.withDocumentFailureOnReplica = withDocumentFailureOnReplica;
@@ -505,7 +506,8 @@ protected TestAction(
                 ignore -> ThreadPool.Names.SAME,
                 false,
                 new IndexingPressureService(settings, clusterService),
-                new SystemIndices(emptyMap())
+                new SystemIndices(emptyMap()),
+                NoopTracer.INSTANCE
             );
             this.withDocumentFailureOnPrimary = false;
             this.withDocumentFailureOnReplica = false;
diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java
index d9bca55a208c2..63a9ac2f2e8ec 100644
--- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java
@@ -125,7 +125,8 @@ public void testRetentionLeaseSyncActionOnPrimary() {
             shardStateAction,
             new ActionFilters(Collections.emptySet()),
             new IndexingPressureService(Settings.EMPTY, clusterService),
-            new SystemIndices(emptyMap())
+            new SystemIndices(emptyMap()),
+            NoopTracer.INSTANCE
         );
         final RetentionLeases retentionLeases = mock(RetentionLeases.class);
         final RetentionLeaseSyncAction.Request request = new RetentionLeaseSyncAction.Request(indexShard.shardId(), retentionLeases);
@@ -162,7 +163,8 @@ public void testRetentionLeaseSyncActionOnReplica() throws Exception {
             shardStateAction,
             new ActionFilters(Collections.emptySet()),
             new IndexingPressureService(Settings.EMPTY, clusterService),
-            new SystemIndices(emptyMap())
+            new SystemIndices(emptyMap()),
+            NoopTracer.INSTANCE
         );
         final RetentionLeases retentionLeases = mock(RetentionLeases.class);
         final RetentionLeaseSyncAction.Request request = new RetentionLeaseSyncAction.Request(indexShard.shardId(), retentionLeases);
@@ -203,7 +205,8 @@ public void testBlocks() {
             shardStateAction,
             new ActionFilters(Collections.emptySet()),
             new IndexingPressureService(Settings.EMPTY, clusterService),
-            new SystemIndices(emptyMap())
+            new SystemIndices(emptyMap()),
+            NoopTracer.INSTANCE
         );
 
         assertNull(action.indexBlockLevel());
@@ -233,7 +236,8 @@ private RetentionLeaseSyncAction createAction() {
             shardStateAction,
             new ActionFilters(Collections.emptySet()),
             new IndexingPressureService(Settings.EMPTY, clusterService),
-            new SystemIndices(emptyMap())
+            new SystemIndices(emptyMap()),
+            NoopTracer.INSTANCE
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
index 2f9f38d18a064..710717532ceb4 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
@@ -2124,7 +2124,8 @@ public void onFailure(final Exception e) {
                             shardStateAction,
                             actionFilters,
                             new IndexingPressureService(settings, clusterService),
-                            new SystemIndices(emptyMap())
+                            new SystemIndices(emptyMap()),
+                            NoopTracer.INSTANCE
                         )
                     ),
                     new GlobalCheckpointSyncAction(
@@ -2190,7 +2191,8 @@ public void onFailure(final Exception e) {
                         mock(ThreadPool.class)
                     ),
                     mock(RemoteStorePressureService.class),
-                    new SystemIndices(emptyMap())
+                    new SystemIndices(emptyMap()),
+                    NoopTracer.INSTANCE
                 );
                 actions.put(
                     BulkAction.INSTANCE,
@@ -2214,7 +2216,8 @@ public void onFailure(final Exception e) {
                         new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, new SystemIndices(emptyMap())),
                         new IndexingPressureService(settings, clusterService),
                         mock(IndicesService.class),
-                        new SystemIndices(emptyMap())
+                        new SystemIndices(emptyMap()),
+                        NoopTracer.INSTANCE
                     )
                 );
                 final RestoreService restoreService = new RestoreService(

From a1fde65fe2ad1acda4364cf5fb751e9b494327ce Mon Sep 17 00:00:00 2001
From: Ashish 
Date: Fri, 20 Oct 2023 21:06:42 +0530
Subject: [PATCH 202/388] [Remote Store] Fix relocation failure due to
 transport receive timeout (#10761)

* [Remote Store] Fix relocation failure due to transport receive timeout

Signed-off-by: Ashish Singh 

* Fix existing extended shardIdle for remote backed shards

Signed-off-by: Ashish Singh 

* Incorporate PR review comments

Signed-off-by: Ashish Singh 

---------

Signed-off-by: Ashish Singh 
---
 .../opensearch/remotestore/RemoteStoreIT.java | 23 +++++++++++++++++++
 .../org/opensearch/index/IndexSettings.java   |  3 +++
 .../opensearch/index/shard/IndexShard.java    |  6 ++++-
 .../translog/InternalTranslogManager.java     | 10 ++++----
 .../index/translog/RemoteFsTranslog.java      |  5 ++++
 .../opensearch/index/translog/Translog.java   |  4 ++++
 .../recovery/PeerRecoverySourceService.java   |  3 ++-
 .../recovery/RemoteRecoveryTargetHandler.java | 13 +++++++++--
 .../index/shard/RemoteIndexShardTests.java    |  9 ++++++++
 .../SegmentReplicationIndexShardTests.java    | 12 ++++++----
 10 files changed, 75 insertions(+), 13 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
index 1fb5c2052aded..b3b4f8e10fd31 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
@@ -509,4 +509,27 @@ public void testRestoreSnapshotToIndexWithSameNameDifferentUUID() throws Excepti
             assertHitCount(client(dataNodes.get(1)).prepareSearch(INDEX_NAME).setSize(0).get(), 50);
         });
     }
+
+    public void testNoSearchIdleForAnyReplicaCount() throws ExecutionException, InterruptedException {
+        internalCluster().startClusterManagerOnlyNode();
+        String primaryShardNode = internalCluster().startDataOnlyNodes(1).get(0);
+
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(0));
+        ensureGreen(INDEX_NAME);
+        IndexShard indexShard = getIndexShard(primaryShardNode);
+        assertFalse(indexShard.isSearchIdleSupported());
+
+        String replicaShardNode = internalCluster().startDataOnlyNodes(1).get(0);
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareUpdateSettings(INDEX_NAME)
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))
+        );
+        ensureGreen(INDEX_NAME);
+        assertFalse(indexShard.isSearchIdleSupported());
+
+        indexShard = getIndexShard(replicaShardNode);
+        assertFalse(indexShard.isSearchIdleSupported());
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index e90e9259f6a5c..99d2b5a74c406 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -1024,6 +1024,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
+        if (this.isRemoteStoreEnabled) {
+            logger.warn("Search idle is not supported for remote backed indices");
+        }
         if (this.replicationType == ReplicationType.SEGMENT && this.getNumberOfReplicas() > 0) {
             logger.warn("Search idle is not supported for indices with replicas using 'replication.type: SEGMENT'");
         }
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 5ebfd3863a6cf..1b7d1b2716979 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -4425,7 +4425,6 @@ public final boolean isSearchIdle() {
     }
 
     /**
-     *
      * Returns true if this shard supports search idle.
      * 
                              
      * Indices using Segment Replication will ignore search idle unless there are no replicas.
@@ -4434,6 +4433,11 @@ public final boolean isSearchIdle() {
      * a new set of segments.
      */
     public final boolean isSearchIdleSupported() {
+        // If the index is remote store backed, then search idle is not supported. This is to ensure that async refresh
+        // task continues to upload to remote store periodically.
+        if (isRemoteTranslogEnabled()) {
+            return false;
+        }
         return indexSettings.isSegRepEnabled() == false || indexSettings.getNumberOfReplicas() == 0;
     }
 
diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
index 85c52b907d326..4d0fc13d433c6 100644
--- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
@@ -430,10 +430,10 @@ public String getTranslogUUID() {
      * @return if the translog should be flushed
      */
     public boolean shouldPeriodicallyFlush(long localCheckpointOfLastCommit, long flushThreshold) {
-        final long translogGenerationOfLastCommit = translog.getMinGenerationForSeqNo(
-            localCheckpointOfLastCommit + 1
-        ).translogFileGeneration;
-        if (translog.sizeInBytesByMinGen(translogGenerationOfLastCommit) < flushThreshold) {
+        // This is the minimum seqNo that is referred in translog and considered for calculating translog size
+        long minTranslogRefSeqNo = translog.getMinUnreferencedSeqNoInSegments(localCheckpointOfLastCommit + 1);
+        final long minReferencedTranslogGeneration = translog.getMinGenerationForSeqNo(minTranslogRefSeqNo).translogFileGeneration;
+        if (translog.sizeInBytesByMinGen(minReferencedTranslogGeneration) < flushThreshold) {
             return false;
         }
         /*
@@ -454,7 +454,7 @@ public boolean shouldPeriodicallyFlush(long localCheckpointOfLastCommit, long fl
         final long translogGenerationOfNewCommit = translog.getMinGenerationForSeqNo(
             localCheckpointTrackerSupplier.get().getProcessedCheckpoint() + 1
         ).translogFileGeneration;
-        return translogGenerationOfLastCommit < translogGenerationOfNewCommit
+        return minReferencedTranslogGeneration < translogGenerationOfNewCommit
             || localCheckpointTrackerSupplier.get().getProcessedCheckpoint() == localCheckpointTrackerSupplier.get().getMaxSeqNo();
     }
 
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
index 29c825fd383c5..2dd9b1a545d4a 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -544,4 +544,9 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) thro
             }
         }
     }
+
+    @Override
+    public long getMinUnreferencedSeqNoInSegments(long minUnrefCheckpointInLastCommit) {
+        return minSeqNoToKeep;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java
index cf7f18840a03e..b44aa6e059224 100644
--- a/server/src/main/java/org/opensearch/index/translog/Translog.java
+++ b/server/src/main/java/org/opensearch/index/translog/Translog.java
@@ -2034,4 +2034,8 @@ public static String createEmptyTranslog(
         writer.close();
         return uuid;
     }
+
+    public long getMinUnreferencedSeqNoInSegments(long minUnrefCheckpointInLastCommit) {
+        return minUnrefCheckpointInLastCommit;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java
index 6c7632a8a408d..cb2bedf00de99 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java
@@ -376,7 +376,8 @@ private Tuple createRecovery
                     transportService,
                     request.targetNode(),
                     recoverySettings,
-                    throttleTime -> shard.recoveryStats().addThrottleTime(throttleTime)
+                    throttleTime -> shard.recoveryStats().addThrottleTime(throttleTime),
+                    shard.isRemoteTranslogEnabled()
                 );
                 handler = RecoverySourceHandlerFactory.create(shard, recoveryTarget, request, recoverySettings);
                 return Tuple.tuple(handler, recoveryTarget);
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
index 66f5b13449f05..37227596fdfe7 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
@@ -75,6 +75,7 @@ public class RemoteRecoveryTargetHandler implements RecoveryTargetHandler {
     private final AtomicLong requestSeqNoGenerator = new AtomicLong(0);
     private final RetryableTransportClient retryableTransportClient;
     private final RemoteSegmentFileChunkWriter fileChunkWriter;
+    private final boolean remoteStoreEnabled;
 
     public RemoteRecoveryTargetHandler(
         long recoveryId,
@@ -82,7 +83,8 @@ public RemoteRecoveryTargetHandler(
         TransportService transportService,
         DiscoveryNode targetNode,
         RecoverySettings recoverySettings,
-        Consumer onSourceThrottle
+        Consumer onSourceThrottle,
+        boolean remoteStoreEnabled
     ) {
         this.transportService = transportService;
         // It is safe to pass the retry timeout value here because RemoteRecoveryTargetHandler
@@ -111,6 +113,7 @@ public RemoteRecoveryTargetHandler(
             requestSeqNoGenerator,
             onSourceThrottle
         );
+        this.remoteStoreEnabled = remoteStoreEnabled;
     }
 
     public DiscoveryNode targetNode() {
@@ -129,7 +132,13 @@ public void prepareForTranslogOperations(int totalTranslogOps, ActionListener reader = in -> TransportResponse.Empty.INSTANCE;
         final ActionListener responseListener = ActionListener.map(listener, r -> null);
-        retryableTransportClient.executeRetryableAction(action, request, responseListener, reader);
+        if (remoteStoreEnabled) {
+            // If remote store is enabled, during the prepare_translog phase, translog is also downloaded on the
+            // target host along with incremental segments download.
+            retryableTransportClient.executeRetryableAction(action, request, translogOpsRequestOptions, responseListener, reader);
+        } else {
+            retryableTransportClient.executeRetryableAction(action, request, responseListener, reader);
+        }
     }
 
     @Override
diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
index fe389e3b3fcb4..703a7d457d5b6 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
@@ -471,6 +471,15 @@ public void onReplicationFailure(
         }
     }
 
+    @Override
+    protected void validateShardIdleWithNoReplicas(IndexShard primary) {
+        // ensure search idle conditions are met.
+        assertFalse(primary.isSearchIdleSupported());
+        assertTrue(primary.isSearchIdle());
+        assertTrue(primary.scheduledRefresh());
+        assertFalse(primary.hasRefreshPending());
+    }
+
     private void assertSingleSegmentFile(IndexShard shard, String fileName) throws IOException {
         final Set segmentsFileNames = Arrays.stream(shard.store().directory().listAll())
             .filter(file -> file.startsWith(IndexFileNames.SEGMENTS))
diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
index eab38bfe5c64d..7caff3e5f5479 100644
--- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
@@ -436,13 +436,17 @@ public void testShardIdleWithNoReplicas() throws Exception {
             shards.startAll();
             final IndexShard primary = shards.getPrimary();
             shards.indexDocs(randomIntBetween(1, 10));
-            // ensure search idle conditions are met.
-            assertTrue(primary.isSearchIdle());
-            assertFalse(primary.scheduledRefresh());
-            assertTrue(primary.hasRefreshPending());
+            validateShardIdleWithNoReplicas(primary);
         }
     }
 
+    protected void validateShardIdleWithNoReplicas(IndexShard primary) {
+        // ensure search idle conditions are met.
+        assertTrue(primary.isSearchIdle());
+        assertFalse(primary.scheduledRefresh());
+        assertTrue(primary.hasRefreshPending());
+    }
+
     /**
      * here we are starting a new primary shard in PrimaryMode and testing if the shard publishes checkpoint after refresh.
      */

From ffe9371b38f98a2c9314bb6cfed4f8f224ff085c Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Fri, 20 Oct 2023 12:40:04 -0400
Subject: [PATCH 203/388] Update Github pull request template to have a task
 for inspecting failing checks (#10751)

* Update Github pull request template to have a task for inspecting failing checks

Signed-off-by: Andriy Redko 

* Add failing builds troubleshooting section to CONTRIBUTING.md

Signed-off-by: Andriy Redko 

* Address review comments

Signed-off-by: Andriy Redko 

---------

Signed-off-by: Andriy Redko 
---
 .github/pull_request_template.md |  1 +
 CONTRIBUTING.md                  | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index c47b9e0b69256..908a032bf833e 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -17,6 +17,7 @@ Resolves #[Issue number to be closed when this PR is merged]
   - [ ] All tests pass
 - [ ] New functionality has been documented.
   - [ ] New functionality has javadoc added
+- [ ] Failing checks are inspected and point to the corresponding known issue(s) (See: [Troubleshooting Failing Builds](../blob/main/CONTRIBUTING.md#troubleshooting-failing-builds))
 - [ ] Commits are signed per the DCO using --signoff
 - [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
 - [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d379d78829318..4a1162cf2558b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -8,6 +8,7 @@
   - [Developer Certificate of Origin](#developer-certificate-of-origin)
   - [Changelog](#changelog)
   - [Review Process](#review-process)
+  - [Troubleshooting Failing Builds](#troubleshooting-failing-builds)
 
 # Contributing to OpenSearch
 
@@ -162,3 +163,14 @@ During the PR process, expect that there will be some back-and-forth. Please try
 If we accept the PR, a [maintainer](MAINTAINERS.md) will merge your change and usually take care of backporting it to appropriate branches ourselves.
 
 If we reject the PR, we will close the pull request with a comment explaining why. This decision isn't always final: if you feel we have misunderstood your intended change or otherwise think that we should reconsider then please continue the conversation with a comment on the PR and we'll do our best to address any further points you raise.
+
+## Troubleshooting Failing Builds
+
+The OpenSearch testing framework offers many capabilities but exhibits significant complexity (it does lot of randomization internally to cover as many edge cases and variations as possible). Unfortunately, this posses a challenge by making it harder to discover important issues/bugs in straightforward way and may lead to so called flaky tests - the tests which flip randomly from success to failure without any code changes. 
+
+If your pull request reports a failing test(s) on one of the checks, please:
+ - look if there is an existing [issue](https://github.com/opensearch-project/OpenSearch/issues) reported for the test in question 
+ - if not, please make sure this is not caused by your changes, run the failing test(s) locally for some time
+ - if you are sure the failure is not related, please open a new [bug](https://github.com/opensearch-project/OpenSearch/issues/new?assignees=&labels=bug%2C+untriaged&projects=&template=bug_template.md&title=%5BBUG%5D) with `flaky-test` label
+ - add a comment referencing the issue(s) or bug report(s) to your pull request explaining the failing build(s)
+ - as a bonus point, try to contribute by fixing the flaky test(s)

From 1e28738b8c966011bf1ae1f00431f0377761cb0a Mon Sep 17 00:00:00 2001
From: Andrew Ross 
Date: Fri, 20 Oct 2023 12:17:51 -0500
Subject: [PATCH 204/388] Increase remote recovery thread pool size (#10750)

The remote recovery thread pool does blocking I/O when downloading
files, so the "half processor count max 10" was definitely too small.
This can be shown by triggering recoveries on a node that is also doing
segment replication, and the replication lag will increase due to
contention on that thread pool. Some amount of contention is inevitable,
but the change here to increase the download thread pool, and also limit
the concurrent usage of that thread pool by any single
recovery/replication to 25% of the threads does help.

Long term, we can improve this even further by moving to fully async I/O
to avoid blocking threads in the application on draining InputStreams.

Signed-off-by: Andrew Ross 
---
 .../org/opensearch/indices/recovery/RecoverySettings.java  | 7 ++++---
 .../main/java/org/opensearch/threadpool/ThreadPool.java    | 7 ++++++-
 .../org/opensearch/threadpool/ScalingThreadPoolTests.java  | 2 +-
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
index 44dfb2f4cb00a..0f3025369833d 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
@@ -41,6 +41,7 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
@@ -87,10 +88,10 @@ public class RecoverySettings {
     /**
      * Controls the maximum number of streams that can be started concurrently per recovery when downloading from the remote store.
      */
-    public static final Setting INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING = Setting.intSetting(
+    public static final Setting INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING = new Setting<>(
         "indices.recovery.max_concurrent_remote_store_streams",
-        10,
-        1,
+        (s) -> Integer.toString(Math.max(1, OpenSearchExecutors.allocatedProcessors(s) / 2)),
+        (s) -> Setting.parseInt(s, 1, "indices.recovery.max_concurrent_remote_store_streams"),
         Property.Dynamic,
         Property.NodeScope
     );
diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
index fab7620292dd2..5f10986239300 100644
--- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
@@ -273,7 +273,12 @@ public ThreadPool(
         );
         builders.put(
             Names.REMOTE_RECOVERY,
-            new ScalingExecutorBuilder(Names.REMOTE_RECOVERY, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5))
+            new ScalingExecutorBuilder(
+                Names.REMOTE_RECOVERY,
+                1,
+                twiceAllocatedProcessors(allocatedProcessors),
+                TimeValue.timeValueMinutes(5)
+            )
         );
         if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) {
             builders.put(
diff --git a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java
index ba2d4b8c247bb..19271bbf30e80 100644
--- a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java
+++ b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java
@@ -154,7 +154,7 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso
         sizes.put(ThreadPool.Names.TRANSLOG_SYNC, n -> 4 * n);
         sizes.put(ThreadPool.Names.REMOTE_PURGE, ThreadPool::halfAllocatedProcessorsMaxFive);
         sizes.put(ThreadPool.Names.REMOTE_REFRESH_RETRY, ThreadPool::halfAllocatedProcessorsMaxTen);
-        sizes.put(ThreadPool.Names.REMOTE_RECOVERY, ThreadPool::halfAllocatedProcessorsMaxTen);
+        sizes.put(ThreadPool.Names.REMOTE_RECOVERY, ThreadPool::twiceAllocatedProcessors);
         return sizes.get(threadPoolName).apply(numberOfProcessors);
     }
 

From 4f8bcff554fe3b019d4149ddcdcc634120c67dd3 Mon Sep 17 00:00:00 2001
From: Poojita Raj 
Date: Fri, 20 Oct 2023 15:57:17 -0700
Subject: [PATCH 205/388] Return unformatted Segment Replication metrics that
 take upload time into account for replication lag (#10723)

* Return unformatted segrep metrics in stats

Signed-off-by: Poojita Raj 

* Take upload time into account for replication time lag

Signed-off-by: Poojita Raj 

* unformat segrep stats

Signed-off-by: Poojita Raj 

* remove unused field names

Signed-off-by: Poojita Raj 

---------

Signed-off-by: Poojita Raj 
---
 .../main/java/org/opensearch/index/ReplicationStats.java  | 8 +++-----
 .../main/java/org/opensearch/index/shard/IndexShard.java  | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/ReplicationStats.java b/server/src/main/java/org/opensearch/index/ReplicationStats.java
index 9cc6685c75f80..0ae4526365bf1 100644
--- a/server/src/main/java/org/opensearch/index/ReplicationStats.java
+++ b/server/src/main/java/org/opensearch/index/ReplicationStats.java
@@ -8,11 +8,9 @@
 
 package org.opensearch.index;
 
-import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 
@@ -76,9 +74,9 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.SEGMENT_REPLICATION);
-        builder.field(Fields.MAX_BYTES_BEHIND, new ByteSizeValue(maxBytesBehind).toString());
-        builder.field(Fields.TOTAL_BYTES_BEHIND, new ByteSizeValue(totalBytesBehind).toString());
-        builder.field(Fields.MAX_REPLICATION_LAG, new TimeValue(maxReplicationLag));
+        builder.field(Fields.MAX_BYTES_BEHIND, maxBytesBehind);
+        builder.field(Fields.TOTAL_BYTES_BEHIND, totalBytesBehind);
+        builder.field(Fields.MAX_REPLICATION_LAG, maxReplicationLag);
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 1b7d1b2716979..f990a3b56e856 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -3010,7 +3010,7 @@ public ReplicationStats getReplicationStats() {
             long maxBytesBehind = stats.stream().mapToLong(SegmentReplicationShardStats::getBytesBehindCount).max().orElse(0L);
             long totalBytesBehind = stats.stream().mapToLong(SegmentReplicationShardStats::getBytesBehindCount).sum();
             long maxReplicationLag = stats.stream()
-                .mapToLong(SegmentReplicationShardStats::getCurrentReplicationTimeMillis)
+                .mapToLong(SegmentReplicationShardStats::getCurrentReplicationLagMillis)
                 .max()
                 .orElse(0L);
             return new ReplicationStats(maxBytesBehind, totalBytesBehind, maxReplicationLag);

From 1e9ec52dd9d5e7d4a3ffb7d37c8b7fdf4069c26d Mon Sep 17 00:00:00 2001
From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com>
Date: Fri, 20 Oct 2023 16:10:31 -0700
Subject: [PATCH 206/388] [Segment Replication] Fix Flaky test
 SegmentReplicationRelocationIT.testPrimaryRelocation (#10701)

* Add primary mode check before assserting on primary mode.

Signed-off-by: Rishikesh1159 

* remove unnecessary shardRouting check.

Signed-off-by: Rishikesh1159 

* Add test logging.

Signed-off-by: Rishikesh1159 

* Addressing comments on PR.

Signed-off-by: Rishikesh1159 

---------

Signed-off-by: Rishikesh1159 
---
 .../replication/SegmentReplicationRelocationIT.java      | 2 ++
 .../replication/SegmentReplicationSourceHandler.java     | 9 ---------
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java
index dd832a63d1e66..dbe0b43441f54 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java
@@ -26,6 +26,7 @@
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.junit.annotations.TestLogging;
 import org.opensearch.test.transport.MockTransportService;
 import org.opensearch.transport.TransportService;
 
@@ -55,6 +56,7 @@ private void createIndex(int replicaCount) {
      * This test verifies happy path when primary shard is relocated newly added node (target) in the cluster. Before
      * relocation and after relocation documents are indexed and documents are verified
      */
+    @TestLogging(reason = "Getting trace logs from replication,shard and allocation package", value = "org.opensearch.indices.replication:TRACE, org.opensearch.index.shard:TRACE, org.opensearch.cluster.routing.allocation:TRACE")
     public void testPrimaryRelocation() throws Exception {
         final String oldPrimary = internalCluster().startNode();
         createIndex(1);
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java
index e2c47b0fb3159..674c09311c645 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java
@@ -12,8 +12,6 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.StepListener;
 import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.cluster.routing.IndexShardRoutingTable;
-import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.util.CancellableThreads;
 import org.opensearch.common.util.concurrent.ListenableFuture;
@@ -22,7 +20,6 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.store.StoreFileMetadata;
-import org.opensearch.indices.recovery.DelayRecoveryException;
 import org.opensearch.indices.recovery.FileChunkWriter;
 import org.opensearch.indices.recovery.MultiChunkTransfer;
 import org.opensearch.indices.replication.common.CopyState;
@@ -146,12 +143,6 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene
                 );
             };
             cancellableThreads.checkForCancel();
-            final IndexShardRoutingTable routingTable = shard.getReplicationGroup().getRoutingTable();
-            ShardRouting targetShardRouting = routingTable.getByAllocationId(request.getTargetAllocationId());
-            if (targetShardRouting == null) {
-                logger.debug("delaying replication of {} as it is not listed as assigned to target node {}", shard.shardId(), targetNode);
-                throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node");
-            }
 
             final StepListener sendFileStep = new StepListener<>();
             Set storeFiles = new HashSet<>(Arrays.asList(shard.store().directory().listAll()));

From 51626d03f857ca840280d1e57cfb1bdfbba75e2d Mon Sep 17 00:00:00 2001
From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com>
Date: Fri, 20 Oct 2023 17:13:43 -0700
Subject: [PATCH 207/388] [Segment Replication] Add Segment Replication
 backpressure rejection stats to _nodes/stats (#10656)

* Initial WIP for adding segrep backpressure to node stats.

Signed-off-by: Rishikesh1159 

* Bind SegmentReplicarionStatsTracker in Node.java

Signed-off-by: Rishikesh1159 

* remove additional segrep backpressure info from node stats

Signed-off-by: Rishikesh1159 

* fix metric name in node stats

Signed-off-by: Rishikesh1159 

* Fix compile error.

Signed-off-by: Rishikesh1159 

* Fix compile errors.

Signed-off-by: Rishikesh1159 

* Address comments on PR.

Signed-off-by: Rishikesh1159 

* Update java docs.

Signed-off-by: Rishikesh1159 

* Address comments on PR and fix compile errors.

Signed-off-by: Rishikesh1159 

* Address comments on PR.

Signed-off-by: Rishikesh1159 

* Update unit test.

Signed-off-by: Rishikesh1159 

---------

Signed-off-by: Rishikesh1159 
Signed-off-by: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com>
---
 .../admin/cluster/node/stats/NodeStats.java   | 24 +++++++
 .../cluster/node/stats/NodesStatsRequest.java |  1 +
 .../node/stats/TransportNodesStatsAction.java |  1 +
 .../stats/TransportClusterStatsAction.java    |  1 +
 .../SegmentReplicationPressureService.java    |  3 +-
 .../SegmentReplicationRejectionStats.java     | 67 +++++++++++++++++++
 .../index/SegmentReplicationStatsTracker.java |  8 +++
 .../main/java/org/opensearch/node/Node.java   |  4 ++
 .../java/org/opensearch/node/NodeService.java |  7 ++
 .../cluster/node/stats/NodeStatsTests.java    | 18 +++++
 .../opensearch/cluster/DiskUsageTests.java    |  6 ++
 ...egmentReplicationPressureServiceTests.java |  9 ++-
 .../SegmentReplicationStatsTrackerTests.java  | 35 ++++++++++
 .../snapshots/SnapshotResiliencyTests.java    |  2 +
 .../MockInternalClusterInfoService.java       |  1 +
 .../opensearch/test/InternalTestCluster.java  |  1 +
 16 files changed, 186 insertions(+), 2 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java
 create mode 100644 server/src/test/java/org/opensearch/index/SegmentReplicationStatsTrackerTests.java

diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index e9bfa358103c8..3d37056956c69 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -46,6 +46,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.discovery.DiscoveryStats;
 import org.opensearch.http.HttpStats;
+import org.opensearch.index.SegmentReplicationRejectionStats;
 import org.opensearch.index.stats.IndexingPressureStats;
 import org.opensearch.index.stats.ShardIndexingPressureStats;
 import org.opensearch.index.store.remote.filecache.FileCacheStats;
@@ -129,6 +130,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private SearchBackpressureStats searchBackpressureStats;
 
+    @Nullable
+    private SegmentReplicationRejectionStats segmentReplicationRejectionStats;
+
     @Nullable
     private ClusterManagerThrottlingStats clusterManagerThrottlingStats;
 
@@ -211,6 +215,12 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             resourceUsageStats = null;
         }
+        // TODO: change to V_2_12_0 on main after backport to 2.x
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            segmentReplicationRejectionStats = in.readOptionalWriteable(SegmentReplicationRejectionStats::new);
+        } else {
+            segmentReplicationRejectionStats = null;
+        }
         if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             repositoriesStats = in.readOptionalWriteable(RepositoriesStats::new);
         } else {
@@ -244,6 +254,7 @@ public NodeStats(
         @Nullable FileCacheStats fileCacheStats,
         @Nullable TaskCancellationStats taskCancellationStats,
         @Nullable SearchPipelineStats searchPipelineStats,
+        @Nullable SegmentReplicationRejectionStats segmentReplicationRejectionStats,
         @Nullable RepositoriesStats repositoriesStats
     ) {
         super(node);
@@ -271,6 +282,7 @@ public NodeStats(
         this.fileCacheStats = fileCacheStats;
         this.taskCancellationStats = taskCancellationStats;
         this.searchPipelineStats = searchPipelineStats;
+        this.segmentReplicationRejectionStats = segmentReplicationRejectionStats;
         this.repositoriesStats = repositoriesStats;
     }
 
@@ -415,6 +427,10 @@ public SearchPipelineStats getSearchPipelineStats() {
     }
 
     @Nullable
+    public SegmentReplicationRejectionStats getSegmentReplicationRejectionStats() {
+        return segmentReplicationRejectionStats;
+    }
+
     public RepositoriesStats getRepositoriesStats() {
         return repositoriesStats;
     }
@@ -465,6 +481,10 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(resourceUsageStats);
         }
+        // TODO: change to V_2_12_0 on main after backport to 2.x
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalWriteable(segmentReplicationRejectionStats);
+        }
         if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(repositoriesStats);
         }
@@ -561,6 +581,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getResourceUsageStats() != null) {
             getResourceUsageStats().toXContent(builder, params);
         }
+        if (getSegmentReplicationRejectionStats() != null) {
+            getSegmentReplicationRejectionStats().toXContent(builder, params);
+        }
+
         if (getRepositoriesStats() != null) {
             getRepositoriesStats().toXContent(builder, params);
         }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index 88dff20354aa2..fc72668d36413 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -215,6 +215,7 @@ public enum Metric {
         TASK_CANCELLATION("task_cancellation"),
         SEARCH_PIPELINE("search_pipeline"),
         RESOURCE_USAGE_STATS("resource_usage_stats"),
+        SEGMENT_REPLICATION_BACKPRESSURE("segment_replication_backpressure"),
         REPOSITORIES("repositories");
 
         private String metricName;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index aa02f8e580f4a..99cf42cfdc4d0 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -126,6 +126,7 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.TASK_CANCELLATION.containedIn(metrics),
             NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics),
             NodesStatsRequest.Metric.RESOURCE_USAGE_STATS.containedIn(metrics),
+            NodesStatsRequest.Metric.SEGMENT_REPLICATION_BACKPRESSURE.containedIn(metrics),
             NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics)
         );
     }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index f51fabbfb2388..5efec8b876435 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -170,6 +170,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List shardsStats = new ArrayList<>();
diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java
index 4284daf9ffef4..d9d480e7b2b27 100644
--- a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java
@@ -106,10 +106,11 @@ public SegmentReplicationPressureService(
         ClusterService clusterService,
         IndicesService indicesService,
         ShardStateAction shardStateAction,
+        SegmentReplicationStatsTracker tracker,
         ThreadPool threadPool
     ) {
         this.indicesService = indicesService;
-        this.tracker = new SegmentReplicationStatsTracker(this.indicesService);
+        this.tracker = tracker;
         this.shardStateAction = shardStateAction;
         this.threadPool = threadPool;
 
diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java
new file mode 100644
index 0000000000000..9f9f150ebe2d7
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java
@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index;
+
+import org.opensearch.Version;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Segment replication rejection stats.
+ *
+ * @opensearch.internal
+ */
+public class SegmentReplicationRejectionStats implements Writeable, ToXContentFragment {
+
+    /**
+     * Total rejections due to segment replication backpressure
+     */
+    private long totalRejectionCount;
+
+    public SegmentReplicationRejectionStats(final long totalRejectionCount) {
+        this.totalRejectionCount = totalRejectionCount;
+    }
+
+    public SegmentReplicationRejectionStats(StreamInput in) throws IOException {
+        // TODO: change to V_2_12_0 on main after backport to 2.x
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            this.totalRejectionCount = in.readVLong();
+        }
+    }
+
+    public long getTotalRejectionCount() {
+        return totalRejectionCount;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject("segment_replication_backpressure");
+        builder.field("total_rejected_requests", totalRejectionCount);
+        return builder.endObject();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // TODO: change to V_2_12_0 on main after backport to 2.x
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeVLong(totalRejectionCount);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "SegmentReplicationRejectionStats{ totalRejectedRequestCount=" + totalRejectionCount + '}';
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java b/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java
index 6d5c00c08caff..f5fc8aa1c1eea 100644
--- a/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationStatsTracker.java
@@ -33,6 +33,14 @@ public SegmentReplicationStatsTracker(IndicesService indicesService) {
         rejectionCount = ConcurrentCollections.newConcurrentMap();
     }
 
+    public SegmentReplicationRejectionStats getTotalRejectionStats() {
+        return new SegmentReplicationRejectionStats(this.rejectionCount.values().stream().mapToInt(AtomicInteger::get).sum());
+    }
+
+    protected Map getRejectionCount() {
+        return rejectionCount;
+    }
+
     public SegmentReplicationStats getStats() {
         Map stats = new HashMap<>();
         for (IndexService indexService : indicesService) {
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index c9148f382a028..711a90d424ac3 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -136,6 +136,7 @@
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.IndexingPressureService;
+import org.opensearch.index.SegmentReplicationStatsTracker;
 import org.opensearch.index.analysis.AnalysisRegistry;
 import org.opensearch.index.engine.EngineFactory;
 import org.opensearch.index.recovery.RemoteStoreRestoreService;
@@ -977,6 +978,7 @@ protected Node(
                 transportService.getTaskManager()
             );
 
+            final SegmentReplicationStatsTracker segmentReplicationStatsTracker = new SegmentReplicationStatsTracker(indicesService);
             RepositoriesModule repositoriesModule = new RepositoriesModule(
                 this.environment,
                 pluginsService.filterPlugins(RepositoryPlugin.class),
@@ -1116,6 +1118,7 @@ protected Node(
                 fileCache,
                 taskCancellationMonitoringService,
                 resourceUsageCollectorService,
+                segmentReplicationStatsTracker,
                 repositoryService
             );
 
@@ -1246,6 +1249,7 @@ protected Node(
                 b.bind(MetricsRegistry.class).toInstance(metricsRegistry);
                 b.bind(RemoteClusterStateService.class).toProvider(() -> remoteClusterStateService);
                 b.bind(PersistedStateRegistry.class).toInstance(persistedStateRegistry);
+                b.bind(SegmentReplicationStatsTracker.class).toInstance(segmentReplicationStatsTracker);
             });
             injector = modules.createInjector();
 
diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java
index e2d7bc2c86ba3..49dde0b81cac7 100644
--- a/server/src/main/java/org/opensearch/node/NodeService.java
+++ b/server/src/main/java/org/opensearch/node/NodeService.java
@@ -48,6 +48,7 @@
 import org.opensearch.discovery.Discovery;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.index.IndexingPressureService;
+import org.opensearch.index.SegmentReplicationStatsTracker;
 import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.ingest.IngestService;
@@ -96,6 +97,8 @@ public class NodeService implements Closeable {
     private final TaskCancellationMonitoringService taskCancellationMonitoringService;
     private final RepositoriesService repositoriesService;
 
+    private final SegmentReplicationStatsTracker segmentReplicationStatsTracker;
+
     NodeService(
         Settings settings,
         ThreadPool threadPool,
@@ -119,6 +122,7 @@ public class NodeService implements Closeable {
         FileCache fileCache,
         TaskCancellationMonitoringService taskCancellationMonitoringService,
         ResourceUsageCollectorService resourceUsageCollectorService,
+        SegmentReplicationStatsTracker segmentReplicationStatsTracker,
         RepositoriesService repositoriesService
     ) {
         this.settings = settings;
@@ -146,6 +150,7 @@ public class NodeService implements Closeable {
         this.repositoriesService = repositoriesService;
         clusterService.addStateApplier(ingestService);
         clusterService.addStateApplier(searchPipelineService);
+        this.segmentReplicationStatsTracker = segmentReplicationStatsTracker;
     }
 
     public NodeInfo info(
@@ -226,6 +231,7 @@ public NodeStats stats(
         boolean taskCancellation,
         boolean searchPipelineStats,
         boolean resourceUsageStats,
+        boolean segmentReplicationTrackerStats,
         boolean repositoriesStats
     ) {
         // for indices stats we want to include previous allocated shards stats as well (it will
@@ -256,6 +262,7 @@ public NodeStats stats(
             fileCacheStats && fileCache != null ? fileCache.fileCacheStats() : null,
             taskCancellation ? this.taskCancellationMonitoringService.stats() : null,
             searchPipelineStats ? this.searchPipelineService.stats() : null,
+            segmentReplicationTrackerStats ? this.segmentReplicationStatsTracker.getTotalRejectionStats() : null,
             repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null
         );
     }
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 3491f18da9550..ebdd012006fb2 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -49,6 +49,7 @@
 import org.opensearch.discovery.DiscoveryStats;
 import org.opensearch.http.HttpStats;
 import org.opensearch.index.ReplicationStats;
+import org.opensearch.index.SegmentReplicationRejectionStats;
 import org.opensearch.index.remote.RemoteSegmentStats;
 import org.opensearch.index.remote.RemoteTranslogTransferTracker;
 import org.opensearch.index.translog.RemoteTranslogStats;
@@ -417,6 +418,17 @@ public void testSerialization() throws IOException {
                         assertEquals(aResourceUsageStats.getTimestamp(), bResourceUsageStats.getTimestamp());
                     });
                 }
+                SegmentReplicationRejectionStats segmentReplicationRejectionStats = nodeStats.getSegmentReplicationRejectionStats();
+                SegmentReplicationRejectionStats deserializedSegmentReplicationRejectionStats = deserializedNodeStats
+                    .getSegmentReplicationRejectionStats();
+                if (segmentReplicationRejectionStats == null) {
+                    assertNull(deserializedSegmentReplicationRejectionStats);
+                } else {
+                    assertEquals(
+                        segmentReplicationRejectionStats.getTotalRejectionCount(),
+                        deserializedSegmentReplicationRejectionStats.getTotalRejectionCount()
+                    );
+                }
                 ScriptCacheStats scriptCacheStats = nodeStats.getScriptCacheStats();
                 ScriptCacheStats deserializedScriptCacheStats = deserializedNodeStats.getScriptCacheStats();
                 if (scriptCacheStats == null) {
@@ -812,6 +824,11 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
             }
             nodesResourceUsageStats = new NodesResourceUsageStats(resourceUsageStatsMap);
         }
+        SegmentReplicationRejectionStats segmentReplicationRejectionStats = null;
+        if (frequently()) {
+            segmentReplicationRejectionStats = new SegmentReplicationRejectionStats(randomNonNegativeLong());
+        }
+
         ClusterManagerThrottlingStats clusterManagerThrottlingStats = null;
         if (frequently()) {
             clusterManagerThrottlingStats = new ClusterManagerThrottlingStats();
@@ -853,6 +870,7 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
             null,
             null,
             null,
+            segmentReplicationRejectionStats,
             null
         );
     }
diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
index 6f03e87bf5824..f037b75dc16a3 100644
--- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
@@ -192,6 +192,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -220,6 +221,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -248,6 +250,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
@@ -307,6 +310,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -335,6 +339,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -363,6 +368,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
diff --git a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java
index 34fa13f0ba62c..478fdcb24f76a 100644
--- a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java
@@ -278,6 +278,13 @@ private SegmentReplicationPressureService buildPressureService(Settings settings
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
 
-        return new SegmentReplicationPressureService(settings, clusterService, indicesService, shardStateAction, mock(ThreadPool.class));
+        return new SegmentReplicationPressureService(
+            settings,
+            clusterService,
+            indicesService,
+            shardStateAction,
+            new SegmentReplicationStatsTracker(indicesService),
+            mock(ThreadPool.class)
+        );
     }
 }
diff --git a/server/src/test/java/org/opensearch/index/SegmentReplicationStatsTrackerTests.java b/server/src/test/java/org/opensearch/index/SegmentReplicationStatsTrackerTests.java
new file mode 100644
index 0000000000000..04423d583e8f9
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/SegmentReplicationStatsTrackerTests.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index;
+
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.test.OpenSearchTestCase;
+
+import org.mockito.Mockito;
+
+import static org.mockito.Mockito.mock;
+
+public class SegmentReplicationStatsTrackerTests extends OpenSearchTestCase {
+
+    private IndicesService indicesService = mock(IndicesService.class);
+
+    public void testRejectedCount() {
+        SegmentReplicationStatsTracker segmentReplicationStatsTracker = new SegmentReplicationStatsTracker(indicesService);
+
+        // Verify that total rejection count is 0 on an empty rejectionCount map in statsTracker.
+        assertTrue(segmentReplicationStatsTracker.getRejectionCount().isEmpty());
+        assertEquals(segmentReplicationStatsTracker.getTotalRejectionStats().getTotalRejectionCount(), 0L);
+
+        // Verify that total rejection count is 1 after incrementing rejectionCount.
+        segmentReplicationStatsTracker.incrementRejectionCount(Mockito.mock(ShardId.class));
+        assertEquals(segmentReplicationStatsTracker.getTotalRejectionStats().getTotalRejectionCount(), 1L);
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
index 710717532ceb4..b7a2baacba611 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
@@ -178,6 +178,7 @@
 import org.opensearch.gateway.TransportNodesListGatewayStartedShards;
 import org.opensearch.index.IndexingPressureService;
 import org.opensearch.index.SegmentReplicationPressureService;
+import org.opensearch.index.SegmentReplicationStatsTracker;
 import org.opensearch.index.analysis.AnalysisRegistry;
 import org.opensearch.index.remote.RemoteStorePressureService;
 import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
@@ -2188,6 +2189,7 @@ public void onFailure(final Exception e) {
                         clusterService,
                         mock(IndicesService.class),
                         mock(ShardStateAction.class),
+                        mock(SegmentReplicationStatsTracker.class),
                         mock(ThreadPool.class)
                     ),
                     mock(RemoteStorePressureService.class),
diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
index 60a54110fd0b4..2ba4de5e54a67 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
@@ -122,6 +122,7 @@ List adjustNodesStats(List nodesStats) {
                 nodeStats.getFileCacheStats(),
                 nodeStats.getTaskCancellationStats(),
                 nodeStats.getSearchPipelineStats(),
+                nodeStats.getSegmentReplicationRejectionStats(),
                 nodeStats.getRepositoriesStats()
             );
         }).collect(Collectors.toList());
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index 898e125b94954..63d8f069bebea 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -2722,6 +2722,7 @@ public void ensureEstimatedStats() {
                     false,
                     false,
                     false,
+                    false,
                     false
                 );
                 assertThat(

From 7c5a806d5bbee77c0c4a184a500bf5522a8d8cd7 Mon Sep 17 00:00:00 2001
From: Movva Ajaykumar 
Date: Sat, 21 Oct 2023 13:31:09 +0530
Subject: [PATCH 208/388] Added changes for AdmissionControl Interceptor and
 AdmissionControlService for RateLimiting (#9286)

* Changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting (#9286)

Signed-off-by: Ajay Kumar Movva 
---
 CHANGELOG.md                                  |   1 +
 .../TransportReplicationAction.java           |  10 +-
 .../common/network/NetworkModule.java         |  16 +-
 .../common/settings/ClusterSettings.java      |   8 +-
 .../main/java/org/opensearch/node/Node.java   |  19 +-
 .../AdmissionControlService.java              | 104 +++++++++
 .../AdmissionControlSettings.java             |  83 ++++++++
 .../controllers/AdmissionController.java      |  70 ++++++
 .../CPUBasedAdmissionController.java          |  55 +++++
 .../controllers/package-info.java             |  12 ++
 .../enums/AdmissionControlMode.java           |  66 ++++++
 .../enums/TransportActionType.java            |  45 ++++
 .../admissioncontrol/enums/package-info.java  |  12 ++
 .../admissioncontrol/package-info.java        |  12 ++
 .../CPUBasedAdmissionControllerSettings.java  | 110 ++++++++++
 .../settings/package-info.java                |  11 +
 .../AdmissionControlTransportHandler.java     |  65 ++++++
 .../AdmissionControlTransportInterceptor.java |  40 ++++
 .../transport/package-info.java               |  11 +
 .../ratelimitting/package-info.java           |  12 ++
 .../common/network/NetworkModuleTests.java    | 199 +++++++++++++++++-
 .../AdmissionControlServiceTests.java         | 140 ++++++++++++
 .../AdmissionControlSettingsTests.java        | 103 +++++++++
 .../CPUBasedAdmissionControllerTests.java     | 109 ++++++++++
 .../enums/AdmissionControlModeTests.java      |  29 +++
 .../enums/TransportActionTypeTests.java       |  27 +++
 ...CPUBasedAdmissionControlSettingsTests.java | 153 ++++++++++++++
 ...AdmissionControlTransportHandlerTests.java |  92 ++++++++
 28 files changed, 1595 insertions(+), 19 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlService.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettings.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/AdmissionController.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionController.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlMode.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionType.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettings.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandler.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportInterceptor.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/package-info.java
 create mode 100644 server/src/main/java/org/opensearch/ratelimitting/package-info.java
 create mode 100644 server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java
 create mode 100644 server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java
 create mode 100644 server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionControllerTests.java
 create mode 100644 server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlModeTests.java
 create mode 100644 server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionTypeTests.java
 create mode 100644 server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControlSettingsTests.java
 create mode 100644 server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandlerTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d7d492679c79d..374dd4ab57ee6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Upload global metadata in cluster state to remote store([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
 - [Remote cluster state] Download functionality of global metadata from remote store ([#10535](https://github.com/opensearch-project/OpenSearch/pull/10535))
 - [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
+- [AdmissionControl] Added changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting ([#9286](https://github.com/opensearch-project/OpenSearch/pull/9286))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
index b68bd13cfed80..ddebdc5530e70 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
@@ -134,6 +134,12 @@ public abstract class TransportReplicationAction<
         Setting.Property.NodeScope
     );
 
+    /**
+     * Making primary and replica actions suffixes as constant
+     */
+    public static final String PRIMARY_ACTION_SUFFIX = "[p]";
+    public static final String REPLICA_ACTION_SUFFIX = "[r]";
+
     protected final ThreadPool threadPool;
     protected final TransportService transportService;
     protected final ClusterService clusterService;
@@ -204,8 +210,8 @@ protected TransportReplicationAction(
         this.shardStateAction = shardStateAction;
         this.executor = executor;
 
-        this.transportPrimaryAction = actionName + "[p]";
-        this.transportReplicaAction = actionName + "[r]";
+        this.transportPrimaryAction = actionName + PRIMARY_ACTION_SUFFIX;
+        this.transportReplicaAction = actionName + REPLICA_ACTION_SUFFIX;
 
         this.initialRetryBackoffBound = REPLICATION_INITIAL_RETRY_BACKOFF_BOUND.get(settings);
         this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings);
diff --git a/server/src/main/java/org/opensearch/common/network/NetworkModule.java b/server/src/main/java/org/opensearch/common/network/NetworkModule.java
index 0734659d8ee72..821d48fccf48c 100644
--- a/server/src/main/java/org/opensearch/common/network/NetworkModule.java
+++ b/server/src/main/java/org/opensearch/common/network/NetworkModule.java
@@ -131,7 +131,7 @@ public final class NetworkModule {
 
     private final Map> transportFactories = new HashMap<>();
     private final Map> transportHttpFactories = new HashMap<>();
-    private final List transportIntercetors = new ArrayList<>();
+    private final List transportInterceptors = new ArrayList<>();
 
     /**
      * Creates a network module that custom networking classes can be plugged into.
@@ -149,9 +149,13 @@ public NetworkModule(
         NetworkService networkService,
         HttpServerTransport.Dispatcher dispatcher,
         ClusterSettings clusterSettings,
-        Tracer tracer
+        Tracer tracer,
+        List transportInterceptors
     ) {
         this.settings = settings;
+        if (transportInterceptors != null) {
+            transportInterceptors.forEach(this::registerTransportInterceptor);
+        }
         for (NetworkPlugin plugin : plugins) {
             Map> httpTransportFactory = plugin.getHttpTransports(
                 settings,
@@ -180,11 +184,11 @@ public NetworkModule(
             for (Map.Entry> entry : transportFactory.entrySet()) {
                 registerTransport(entry.getKey(), entry.getValue());
             }
-            List transportInterceptors = plugin.getTransportInterceptors(
+            List pluginTransportInterceptors = plugin.getTransportInterceptors(
                 namedWriteableRegistry,
                 threadPool.getThreadContext()
             );
-            for (TransportInterceptor interceptor : transportInterceptors) {
+            for (TransportInterceptor interceptor : pluginTransportInterceptors) {
                 registerTransportInterceptor(interceptor);
             }
         }
@@ -264,7 +268,7 @@ public Supplier getTransportSupplier() {
      * Registers a new {@link TransportInterceptor}
      */
     private void registerTransportInterceptor(TransportInterceptor interceptor) {
-        this.transportIntercetors.add(Objects.requireNonNull(interceptor, "interceptor must not be null"));
+        this.transportInterceptors.add(Objects.requireNonNull(interceptor, "interceptor must not be null"));
     }
 
     /**
@@ -272,7 +276,7 @@ private void registerTransportInterceptor(TransportInterceptor interceptor) {
      * @see #registerTransportInterceptor(TransportInterceptor)
      */
     public TransportInterceptor getTransportInterceptor() {
-        return new CompositeTransportInterceptor(this.transportIntercetors);
+        return new CompositeTransportInterceptor(this.transportInterceptors);
     }
 
     static final class CompositeTransportInterceptor implements TransportInterceptor {
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 76883c200542e..7ac7da819b215 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -134,6 +134,8 @@
 import org.opensearch.persistent.PersistentTasksClusterService;
 import org.opensearch.persistent.decider.EnableAssignmentDecider;
 import org.opensearch.plugins.PluginsService;
+import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlSettings;
+import org.opensearch.ratelimitting.admissioncontrol.settings.CPUBasedAdmissionControllerSettings;
 import org.opensearch.repositories.fs.FsRepository;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.script.ScriptService;
@@ -682,7 +684,11 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING,
                 RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
                 IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
-                IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING
+                IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
+                AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
+                CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
+                CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 711a90d424ac3..e80b768074fc7 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -197,6 +197,8 @@
 import org.opensearch.plugins.SearchPlugin;
 import org.opensearch.plugins.SystemIndexPlugin;
 import org.opensearch.plugins.TelemetryPlugin;
+import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
+import org.opensearch.ratelimitting.admissioncontrol.transport.AdmissionControlTransportInterceptor;
 import org.opensearch.repositories.RepositoriesModule;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.rest.RestController;
@@ -820,6 +822,7 @@ protected Node(
                 remoteStoreStatsTrackerFactory,
                 recoverySettings
             );
+
             final AliasValidator aliasValidator = new AliasValidator();
 
             final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService, systemIndices);
@@ -891,6 +894,17 @@ protected Node(
 
             final RestController restController = actionModule.getRestController();
 
+            final AdmissionControlService admissionControlService = new AdmissionControlService(
+                settings,
+                clusterService.getClusterSettings(),
+                threadPool
+            );
+
+            AdmissionControlTransportInterceptor admissionControlTransportInterceptor = new AdmissionControlTransportInterceptor(
+                admissionControlService
+            );
+
+            List transportInterceptors = List.of(admissionControlTransportInterceptor);
             final NetworkModule networkModule = new NetworkModule(
                 settings,
                 pluginsService.filterPlugins(NetworkPlugin.class),
@@ -903,8 +917,10 @@ protected Node(
                 networkService,
                 restController,
                 clusterService.getClusterSettings(),
-                tracer
+                tracer,
+                transportInterceptors
             );
+
             Collection>> indexTemplateMetadataUpgraders = pluginsService.filterPlugins(
                 Plugin.class
             ).stream().map(Plugin::getIndexTemplateMetadataUpgrader).collect(Collectors.toList());
@@ -1181,6 +1197,7 @@ protected Node(
                 b.bind(IndexingPressureService.class).toInstance(indexingPressureService);
                 b.bind(TaskResourceTrackingService.class).toInstance(taskResourceTrackingService);
                 b.bind(SearchBackpressureService.class).toInstance(searchBackpressureService);
+                b.bind(AdmissionControlService.class).toInstance(admissionControlService);
                 b.bind(UsageService.class).toInstance(usageService);
                 b.bind(AggregationUsageService.class).toInstance(searchModule.getValuesSourceRegistry().getUsageService());
                 b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlService.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlService.java
new file mode 100644
index 0000000000000..2cc409b0e4465
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlService.java
@@ -0,0 +1,104 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.controllers.AdmissionController;
+import org.opensearch.ratelimitting.admissioncontrol.controllers.CPUBasedAdmissionController;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.opensearch.ratelimitting.admissioncontrol.settings.CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER;
+
+/**
+ * Admission control Service that bootstraps and manages all the Admission Controllers in OpenSearch.
+ */
+public class AdmissionControlService {
+    private final ThreadPool threadPool;
+    public final AdmissionControlSettings admissionControlSettings;
+    private final ConcurrentMap ADMISSION_CONTROLLERS;
+    private static final Logger logger = LogManager.getLogger(AdmissionControlService.class);
+    private final ClusterSettings clusterSettings;
+    private final Settings settings;
+
+    /**
+     *
+     * @param settings Immutable settings instance
+     * @param clusterSettings ClusterSettings Instance
+     * @param threadPool ThreadPool Instance
+     */
+    public AdmissionControlService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
+        this.threadPool = threadPool;
+        this.admissionControlSettings = new AdmissionControlSettings(clusterSettings, settings);
+        this.ADMISSION_CONTROLLERS = new ConcurrentHashMap<>();
+        this.clusterSettings = clusterSettings;
+        this.settings = settings;
+        this.initialise();
+    }
+
+    /**
+     * Initialise and Register all the admissionControllers
+     */
+    private void initialise() {
+        // Initialise different type of admission controllers
+        registerAdmissionController(CPU_BASED_ADMISSION_CONTROLLER);
+    }
+
+    /**
+     * Handler to trigger registered admissionController
+     */
+    public void applyTransportAdmissionControl(String action) {
+        this.ADMISSION_CONTROLLERS.forEach((name, admissionController) -> { admissionController.apply(action); });
+    }
+
+    /**
+     *
+     * @param admissionControllerName admissionControllerName to register into the service.
+     */
+    public void registerAdmissionController(String admissionControllerName) {
+        AdmissionController admissionController = this.controllerFactory(admissionControllerName);
+        this.ADMISSION_CONTROLLERS.put(admissionControllerName, admissionController);
+    }
+
+    /**
+     * @return AdmissionController Instance
+     */
+    private AdmissionController controllerFactory(String admissionControllerName) {
+        switch (admissionControllerName) {
+            case CPU_BASED_ADMISSION_CONTROLLER:
+                return new CPUBasedAdmissionController(admissionControllerName, this.settings, this.clusterSettings);
+            default:
+                throw new IllegalArgumentException("Not Supported AdmissionController : " + admissionControllerName);
+        }
+    }
+
+    /**
+     *
+     * @return list of the registered admissionControllers
+     */
+    public List getAdmissionControllers() {
+        return new ArrayList<>(this.ADMISSION_CONTROLLERS.values());
+    }
+
+    /**
+     *
+     * @param controllerName name of the admissionController
+     * @return instance of the AdmissionController Instance
+     */
+    public AdmissionController getAdmissionController(String controllerName) {
+        return this.ADMISSION_CONTROLLERS.getOrDefault(controllerName, null);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettings.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettings.java
new file mode 100644
index 0000000000000..b557190ab54ac
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettings.java
@@ -0,0 +1,83 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+
+/**
+ * Settings related to admission control.
+ * @opensearch.internal
+ */
+public final class AdmissionControlSettings {
+
+    /**
+     * Default parameters for the AdmissionControlSettings
+     */
+    public static class Defaults {
+        public static final String MODE = "disabled";
+    }
+
+    /**
+     * Feature level setting to operate in shadow-mode or in enforced-mode. If enforced field is set
+     * rejection will be performed, otherwise only rejection metrics will be populated.
+     */
+    public static final Setting ADMISSION_CONTROL_TRANSPORT_LAYER_MODE = new Setting<>(
+        "admission_control.transport.mode",
+        Defaults.MODE,
+        AdmissionControlMode::fromName,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    private volatile AdmissionControlMode transportLayeradmissionControlMode;
+
+    /**
+     * @param clusterSettings clusterSettings Instance
+     * @param settings        settings instance
+     */
+    public AdmissionControlSettings(ClusterSettings clusterSettings, Settings settings) {
+        this.transportLayeradmissionControlMode = ADMISSION_CONTROL_TRANSPORT_LAYER_MODE.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, this::setAdmissionControlTransportLayerMode);
+    }
+
+    /**
+     *
+     * @param admissionControlMode update the mode of admission control feature
+     */
+    private void setAdmissionControlTransportLayerMode(AdmissionControlMode admissionControlMode) {
+        this.transportLayeradmissionControlMode = admissionControlMode;
+    }
+
+    /**
+     *
+     * @return return the default mode of the admissionControl
+     */
+    public AdmissionControlMode getAdmissionControlTransportLayerMode() {
+        return this.transportLayeradmissionControlMode;
+    }
+
+    /**
+     *
+     * @return true based on the admission control feature is enforced else false
+     */
+    public Boolean isTransportLayerAdmissionControlEnforced() {
+        return this.transportLayeradmissionControlMode == AdmissionControlMode.ENFORCED;
+    }
+
+    /**
+     *
+     * @return true based on the admission control feature is enabled else false
+     */
+    public Boolean isTransportLayerAdmissionControlEnabled() {
+        return this.transportLayeradmissionControlMode != AdmissionControlMode.DISABLED;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/AdmissionController.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/AdmissionController.java
new file mode 100644
index 0000000000000..00564a9967f31
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/AdmissionController.java
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.controllers;
+
+import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Abstract class for Admission Controller in OpenSearch, which aims to provide resource based request admission control.
+ * It provides methods for any tracking-object that can be incremented (such as memory size),
+ * and admission control can be applied if configured limit has been reached
+ */
+public abstract class AdmissionController {
+
+    private final AtomicLong rejectionCount;
+    private final String admissionControllerName;
+
+    /**
+     *
+     * @param rejectionCount initialised rejectionCount value for AdmissionController
+     * @param admissionControllerName name of the admissionController
+     */
+    public AdmissionController(AtomicLong rejectionCount, String admissionControllerName) {
+        this.rejectionCount = rejectionCount;
+        this.admissionControllerName = admissionControllerName;
+    }
+
+    /**
+     * Return the current state of the admission controller
+     * @return true if admissionController is enabled for the transport layer else false
+     */
+    public boolean isEnabledForTransportLayer(AdmissionControlMode admissionControlMode) {
+        return admissionControlMode != AdmissionControlMode.DISABLED;
+    }
+
+    /**
+     * Increment the tracking-objects and apply the admission control if threshold is breached.
+     * Mostly applicable while applying admission controller
+     */
+    public abstract void apply(String action);
+
+    /**
+     * @return name of the admission-controller
+     */
+    public String getName() {
+        return this.admissionControllerName;
+    }
+
+    /**
+     * Adds the rejection count for the controller. Primarily used when copying controller states.
+     * @param count To add the value of the tracking resource object as the provided count
+     */
+    public void addRejectionCount(long count) {
+        this.rejectionCount.addAndGet(count);
+    }
+
+    /**
+     * @return current value of the rejection count metric tracked by the admission-controller.
+     */
+    public long getRejectionCount() {
+        return this.rejectionCount.get();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionController.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionController.java
new file mode 100644
index 0000000000000..3a8956b2cce87
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionController.java
@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.controllers;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.settings.CPUBasedAdmissionControllerSettings;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ *  Class for CPU Based Admission Controller in OpenSearch, which aims to provide CPU utilisation admission control.
+ *  It provides methods to apply admission control if configured limit has been reached
+ */
+public class CPUBasedAdmissionController extends AdmissionController {
+    private static final Logger LOGGER = LogManager.getLogger(CPUBasedAdmissionController.class);
+    public CPUBasedAdmissionControllerSettings settings;
+
+    /**
+     *
+     * @param admissionControllerName State of the admission controller
+     */
+    public CPUBasedAdmissionController(String admissionControllerName, Settings settings, ClusterSettings clusterSettings) {
+        super(new AtomicLong(0), admissionControllerName);
+        this.settings = new CPUBasedAdmissionControllerSettings(clusterSettings, settings);
+    }
+
+    /**
+     * This function will take of applying admission controller based on CPU usage
+     * @param action is the transport action
+     */
+    @Override
+    public void apply(String action) {
+        // TODO Will extend this logic further currently just incrementing rejectionCount
+        if (this.isEnabledForTransportLayer(this.settings.getTransportLayerAdmissionControllerMode())) {
+            this.applyForTransportLayer(action);
+        }
+    }
+
+    private void applyForTransportLayer(String actionName) {
+        // currently incrementing counts to evaluate the controller triggering as expected and using in testing so limiting to 10
+        // TODO will update rejection logic further in next PR's
+        if (this.getRejectionCount() < 10) {
+            this.addRejectionCount(1);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/package-info.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/package-info.java
new file mode 100644
index 0000000000000..23746cc61a203
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/controllers/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * This package contains classes related to the different admission controllers
+ */
+package org.opensearch.ratelimitting.admissioncontrol.controllers;
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlMode.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlMode.java
new file mode 100644
index 0000000000000..2ae2436ba84e7
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlMode.java
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.enums;
+
+import java.util.Locale;
+
+/**
+ * Defines the AdmissionControlMode
+ */
+public enum AdmissionControlMode {
+    /**
+     * AdmissionController is completely disabled.
+     */
+    DISABLED("disabled"),
+
+    /**
+     * AdmissionController only monitors the rejection criteria for the requests.
+     */
+    MONITOR("monitor_only"),
+
+    /**
+     * AdmissionController monitors and rejects tasks that exceed resource usage thresholds.
+     */
+    ENFORCED("enforced");
+
+    private final String mode;
+
+    /**
+     * @param mode update mode of the admission controller
+     */
+    AdmissionControlMode(String mode) {
+        this.mode = mode;
+    }
+
+    /**
+     *
+     * @return mode of the admission controller
+     */
+    public String getMode() {
+        return this.mode;
+    }
+
+    /**
+     *
+     * @param name is the mode of the current
+     * @return Enum of AdmissionControlMode based on the mode
+     */
+    public static AdmissionControlMode fromName(String name) {
+        switch (name.toLowerCase(Locale.ROOT)) {
+            case "disabled":
+                return DISABLED;
+            case "monitor_only":
+                return MONITOR;
+            case "enforced":
+                return ENFORCED;
+            default:
+                throw new IllegalArgumentException("Invalid AdmissionControlMode: " + name);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionType.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionType.java
new file mode 100644
index 0000000000000..f2fdca0cfe49b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionType.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.enums;
+
+import java.util.Locale;
+
+/**
+ * Enums that defines the type of the transport requests
+ */
+public enum TransportActionType {
+    INDEXING("indexing"),
+    SEARCH("search");
+
+    private final String type;
+
+    TransportActionType(String uriType) {
+        this.type = uriType;
+    }
+
+    /**
+     *
+     * @return type of the request
+     */
+    public String getType() {
+        return type;
+    }
+
+    public static TransportActionType fromName(String name) {
+        name = name.toLowerCase(Locale.ROOT);
+        switch (name) {
+            case "indexing":
+                return INDEXING;
+            case "search":
+                return SEARCH;
+            default:
+                throw new IllegalArgumentException("Not Supported TransportAction Type: " + name);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/package-info.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/package-info.java
new file mode 100644
index 0000000000000..98b08ebd0a7bf
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/enums/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * This package contains enums related to the different admission controller feature
+ */
+package org.opensearch.ratelimitting.admissioncontrol.enums;
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/package-info.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/package-info.java
new file mode 100644
index 0000000000000..b3dc229f86fb6
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * This package contains base classes needed for the admissionController Feature
+ */
+package org.opensearch.ratelimitting.admissioncontrol;
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettings.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettings.java
new file mode 100644
index 0000000000000..141e9b68db145
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControllerSettings.java
@@ -0,0 +1,110 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.settings;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlSettings;
+import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Settings related to cpu based admission controller.
+ * @opensearch.internal
+ */
+public class CPUBasedAdmissionControllerSettings {
+    public static final String CPU_BASED_ADMISSION_CONTROLLER = "global_cpu_usage";
+
+    /**
+     * Default parameters for the CPUBasedAdmissionControllerSettings
+     */
+    public static class Defaults {
+        public static final long CPU_USAGE = 95;
+        public static List TRANSPORT_LAYER_DEFAULT_URI_TYPE = Arrays.asList("indexing", "search");
+    }
+
+    private AdmissionControlMode transportLayerMode;
+    private Long searchCPULimit;
+    private Long indexingCPULimit;
+
+    private final List transportActionsList;
+    /**
+     * Feature level setting to operate in shadow-mode or in enforced-mode. If enforced field is set
+     * rejection will be performed, otherwise only rejection metrics will be populated.
+     */
+    public static final Setting CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE = new Setting<>(
+        "admission_control.transport.cpu_usage.mode_override",
+        AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
+        AdmissionControlMode::fromName,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * This setting used to set the CPU Limits for the search requests by default it will use default IO usage limit
+     */
+    public static final Setting SEARCH_CPU_USAGE_LIMIT = Setting.longSetting(
+        "admission_control.search.cpu_usage.limit",
+        Defaults.CPU_USAGE,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * This setting used to set the CPU limits for the indexing requests by default it will use default IO usage limit
+     */
+    public static final Setting INDEXING_CPU_USAGE_LIMIT = Setting.longSetting(
+        "admission_control.indexing.cpu_usage.limit",
+        Defaults.CPU_USAGE,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    // currently limited to one setting will add further more settings in follow-up PR's
+    public CPUBasedAdmissionControllerSettings(ClusterSettings clusterSettings, Settings settings) {
+        this.transportLayerMode = CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE, this::setTransportLayerMode);
+        this.searchCPULimit = SEARCH_CPU_USAGE_LIMIT.get(settings);
+        this.indexingCPULimit = INDEXING_CPU_USAGE_LIMIT.get(settings);
+        this.transportActionsList = Defaults.TRANSPORT_LAYER_DEFAULT_URI_TYPE;
+        clusterSettings.addSettingsUpdateConsumer(INDEXING_CPU_USAGE_LIMIT, this::setIndexingCPULimit);
+        clusterSettings.addSettingsUpdateConsumer(SEARCH_CPU_USAGE_LIMIT, this::setSearchCPULimit);
+    }
+
+    private void setTransportLayerMode(AdmissionControlMode admissionControlMode) {
+        this.transportLayerMode = admissionControlMode;
+    }
+
+    public AdmissionControlMode getTransportLayerAdmissionControllerMode() {
+        return transportLayerMode;
+    }
+
+    public Long getSearchCPULimit() {
+        return searchCPULimit;
+    }
+
+    public Long getIndexingCPULimit() {
+        return indexingCPULimit;
+    }
+
+    public void setIndexingCPULimit(Long indexingCPULimit) {
+        this.indexingCPULimit = indexingCPULimit;
+    }
+
+    public void setSearchCPULimit(Long searchCPULimit) {
+        this.searchCPULimit = searchCPULimit;
+    }
+
+    public List getTransportActionsList() {
+        return transportActionsList;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/package-info.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/package-info.java
new file mode 100644
index 0000000000000..a024ccc756745
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/settings/package-info.java
@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/**
+ * This package contains settings related classes for the different admission controllers
+ */
+package org.opensearch.ratelimitting.admissioncontrol.settings;
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandler.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandler.java
new file mode 100644
index 0000000000000..7d0f5fbc17a51
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandler.java
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.transport;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
+import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
+import org.opensearch.tasks.Task;
+import org.opensearch.transport.TransportChannel;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportRequestHandler;
+
+/**
+ * AdmissionControl Handler to intercept Transport Requests.
+ * @param  Transport Request
+ */
+public class AdmissionControlTransportHandler implements TransportRequestHandler {
+
+    private final String action;
+    private final TransportRequestHandler actualHandler;
+    protected final Logger log = LogManager.getLogger(this.getClass());
+    AdmissionControlService admissionControlService;
+    boolean forceExecution;
+
+    public AdmissionControlTransportHandler(
+        String action,
+        TransportRequestHandler actualHandler,
+        AdmissionControlService admissionControlService,
+        boolean forceExecution
+    ) {
+        super();
+        this.action = action;
+        this.actualHandler = actualHandler;
+        this.admissionControlService = admissionControlService;
+        this.forceExecution = forceExecution;
+    }
+
+    /**
+     * @param request Transport Request that landed on the node
+     * @param channel Transport channel allows to send a response to a request
+     * @param task Current task that is executing
+     * @throws Exception when admission control rejected the requests
+     */
+    @Override
+    public void messageReceived(T request, TransportChannel channel, Task task) throws Exception {
+        // intercept all the transport requests here and apply admission control
+        try {
+            // TODO Need to evaluate if we need to apply admission control or not if force Execution is true will update in next PR.
+            this.admissionControlService.applyTransportAdmissionControl(this.action);
+        } catch (final OpenSearchRejectedExecutionException openSearchRejectedExecutionException) {
+            log.warn(openSearchRejectedExecutionException.getMessage());
+            channel.sendResponse(openSearchRejectedExecutionException);
+        } catch (final Exception e) {
+            throw e;
+        }
+        actualHandler.messageReceived(request, channel, task);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportInterceptor.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportInterceptor.java
new file mode 100644
index 0000000000000..01cfcbd780006
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportInterceptor.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.transport;
+
+import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
+import org.opensearch.transport.TransportInterceptor;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportRequestHandler;
+
+/**
+ * This class allows throttling to intercept requests on both the sender and the receiver side.
+ */
+public class AdmissionControlTransportInterceptor implements TransportInterceptor {
+
+    AdmissionControlService admissionControlService;
+
+    public AdmissionControlTransportInterceptor(AdmissionControlService admissionControlService) {
+        this.admissionControlService = admissionControlService;
+    }
+
+    /**
+     *
+     * @return admissionController handler to intercept transport requests
+     */
+    @Override
+    public  TransportRequestHandler interceptHandler(
+        String action,
+        String executor,
+        boolean forceExecution,
+        TransportRequestHandler actualHandler
+    ) {
+        return new AdmissionControlTransportHandler<>(action, actualHandler, this.admissionControlService, forceExecution);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/package-info.java b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/package-info.java
new file mode 100644
index 0000000000000..f97f31bc7b1db
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/admissioncontrol/transport/package-info.java
@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/**
+ * This package contains transport related classes for the admissionController Feature
+ */
+package org.opensearch.ratelimitting.admissioncontrol.transport;
diff --git a/server/src/main/java/org/opensearch/ratelimitting/package-info.java b/server/src/main/java/org/opensearch/ratelimitting/package-info.java
new file mode 100644
index 0000000000000..c04358e14284f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ratelimitting/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Base OpenSearch Throttling package
+ */
+package org.opensearch.ratelimitting;
diff --git a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
index 0ca118fe422a5..ab51cafb039c2 100644
--- a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
+++ b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
@@ -57,6 +57,7 @@
 import org.opensearch.transport.TransportRequest;
 import org.opensearch.transport.TransportRequestHandler;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -124,7 +125,7 @@ public Map> getTransports(
                 return Collections.singletonMap("custom", custom);
             }
         };
-        NetworkModule module = newNetworkModule(settings, plugin);
+        NetworkModule module = newNetworkModule(settings, null, plugin);
         assertSame(custom, module.getTransportSupplier());
     }
 
@@ -135,7 +136,7 @@ public void testRegisterHttpTransport() {
             .build();
         Supplier custom = FakeHttpTransport::new;
 
-        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, null, new NetworkPlugin() {
             @Override
             public Map> getHttpTransports(
                 Settings settings,
@@ -155,7 +156,7 @@ public Map> getHttpTransports(
         assertSame(custom, module.getHttpServerTransportSupplier());
 
         settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
-        NetworkModule newModule = newNetworkModule(settings);
+        NetworkModule newModule = newNetworkModule(settings, null);
         expectThrows(IllegalStateException.class, () -> newModule.getHttpServerTransportSupplier());
     }
 
@@ -169,7 +170,7 @@ public void testOverrideDefault() {
         Supplier customTransport = () -> null;  // content doesn't matter we check reference equality
         Supplier custom = FakeHttpTransport::new;
         Supplier def = FakeHttpTransport::new;
-        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, null, new NetworkPlugin() {
             @Override
             public Map> getTransports(
                 Settings settings,
@@ -214,7 +215,7 @@ public void testDefaultKeys() {
         Supplier custom = FakeHttpTransport::new;
         Supplier def = FakeHttpTransport::new;
         Supplier customTransport = () -> null;
-        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, null, new NetworkPlugin() {
             @Override
             public Map> getTransports(
                 Settings settings,
@@ -273,7 +274,7 @@ public  TransportRequestHandler interceptHandler(
                 return actualHandler;
             }
         };
-        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, null, new NetworkPlugin() {
             @Override
             public List getTransportInterceptors(
                 NamedWriteableRegistry namedWriteableRegistry,
@@ -295,7 +296,7 @@ public List getTransportInterceptors(
         assertSame(((NetworkModule.CompositeTransportInterceptor) transportInterceptor).transportInterceptors.get(0), interceptor);
 
         NullPointerException nullPointerException = expectThrows(NullPointerException.class, () -> {
-            newNetworkModule(settings, new NetworkPlugin() {
+            newNetworkModule(settings, null, new NetworkPlugin() {
                 @Override
                 public List getTransportInterceptors(
                     NamedWriteableRegistry namedWriteableRegistry,
@@ -309,7 +310,186 @@ public List getTransportInterceptors(
         assertEquals("interceptor must not be null", nullPointerException.getMessage());
     }
 
-    private NetworkModule newNetworkModule(Settings settings, NetworkPlugin... plugins) {
+    public void testRegisterCoreInterceptor() {
+        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
+        AtomicInteger called = new AtomicInteger(0);
+
+        TransportInterceptor interceptor = new TransportInterceptor() {
+            @Override
+            public  TransportRequestHandler interceptHandler(
+                String action,
+                String executor,
+                boolean forceExecution,
+                TransportRequestHandler actualHandler
+            ) {
+                called.incrementAndGet();
+                if ("foo/bar/boom".equals(action)) {
+                    assertTrue(forceExecution);
+                } else {
+                    assertFalse(forceExecution);
+                }
+                return actualHandler;
+            }
+        };
+
+        List coreTransportInterceptors = new ArrayList<>();
+        coreTransportInterceptors.add(interceptor);
+
+        NetworkModule module = newNetworkModule(settings, coreTransportInterceptors);
+
+        TransportInterceptor transportInterceptor = module.getTransportInterceptor();
+        assertEquals(0, called.get());
+        transportInterceptor.interceptHandler("foo/bar/boom", null, true, null);
+        assertEquals(1, called.get());
+        transportInterceptor.interceptHandler("foo/baz/boom", null, false, null);
+        assertEquals(2, called.get());
+        assertTrue(transportInterceptor instanceof NetworkModule.CompositeTransportInterceptor);
+        assertEquals(((NetworkModule.CompositeTransportInterceptor) transportInterceptor).transportInterceptors.size(), 1);
+        assertSame(((NetworkModule.CompositeTransportInterceptor) transportInterceptor).transportInterceptors.get(0), interceptor);
+    }
+
+    public void testInterceptorOrder() {
+        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
+        AtomicInteger called = new AtomicInteger(0);
+        AtomicInteger called1 = new AtomicInteger(0);
+
+        TransportInterceptor interceptor = new TransportInterceptor() {
+            @Override
+            public  TransportRequestHandler interceptHandler(
+                String action,
+                String executor,
+                boolean forceExecution,
+                TransportRequestHandler actualHandler
+            ) {
+                called.incrementAndGet();
+                if ("foo/bar/boom".equals(action)) {
+                    assertTrue(forceExecution);
+                } else {
+                    assertFalse(forceExecution);
+                }
+                return actualHandler;
+            }
+        };
+
+        TransportInterceptor interceptor1 = new TransportInterceptor() {
+            @Override
+            public  TransportRequestHandler interceptHandler(
+                String action,
+                String executor,
+                boolean forceExecution,
+                TransportRequestHandler actualHandler
+            ) {
+                called1.incrementAndGet();
+                if ("foo/bar/boom".equals(action)) {
+                    assertTrue(forceExecution);
+                } else {
+                    assertFalse(forceExecution);
+                }
+                return actualHandler;
+            }
+        };
+
+        List coreTransportInterceptors = new ArrayList<>();
+        coreTransportInterceptors.add(interceptor1);
+
+        NetworkModule module = newNetworkModule(settings, coreTransportInterceptors, new NetworkPlugin() {
+            @Override
+            public List getTransportInterceptors(
+                NamedWriteableRegistry namedWriteableRegistry,
+                ThreadContext threadContext
+            ) {
+                assertNotNull(threadContext);
+                return Collections.singletonList(interceptor);
+            }
+        });
+
+        TransportInterceptor transportInterceptor = module.getTransportInterceptor();
+        assertEquals(((NetworkModule.CompositeTransportInterceptor) transportInterceptor).transportInterceptors.size(), 2);
+
+        assertEquals(0, called.get());
+        assertEquals(0, called1.get());
+        transportInterceptor.interceptHandler("foo/bar/boom", null, true, null);
+        assertEquals(1, called.get());
+        assertEquals(1, called1.get());
+        transportInterceptor.interceptHandler("foo/baz/boom", null, false, null);
+        assertEquals(2, called.get());
+        assertEquals(2, called1.get());
+    }
+
+    public void testInterceptorOrderException() {
+        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
+        AtomicInteger called = new AtomicInteger(0);
+        AtomicInteger called1 = new AtomicInteger(0);
+
+        TransportInterceptor interceptor = new TransportInterceptor() {
+            @Override
+            public  TransportRequestHandler interceptHandler(
+                String action,
+                String executor,
+                boolean forceExecution,
+                TransportRequestHandler actualHandler
+            ) {
+                called.incrementAndGet();
+                if ("foo/bar/boom".equals(action)) {
+                    assertTrue(forceExecution);
+                } else {
+                    assertFalse(forceExecution);
+                }
+                return actualHandler;
+            }
+        };
+
+        TransportInterceptor interceptor1 = new TransportInterceptor() {
+            @Override
+            public  TransportRequestHandler interceptHandler(
+                String action,
+                String executor,
+                boolean forceExecution,
+                TransportRequestHandler actualHandler
+            ) {
+                called1.incrementAndGet();
+                throw new RuntimeException("Handler Invoke Failed");
+            }
+        };
+
+        List coreTransportInterceptors = new ArrayList<>();
+        coreTransportInterceptors.add(interceptor1);
+
+        NetworkModule module = newNetworkModule(settings, coreTransportInterceptors, new NetworkPlugin() {
+            @Override
+            public List getTransportInterceptors(
+                NamedWriteableRegistry namedWriteableRegistry,
+                ThreadContext threadContext
+            ) {
+                assertNotNull(threadContext);
+                return Collections.singletonList(interceptor);
+            }
+        });
+
+        TransportInterceptor transportInterceptor = module.getTransportInterceptor();
+        assertEquals(((NetworkModule.CompositeTransportInterceptor) transportInterceptor).transportInterceptors.size(), 2);
+
+        assertEquals(0, called.get());
+        assertEquals(0, called1.get());
+        try {
+            transportInterceptor.interceptHandler("foo/bar/boom", null, true, null);
+        } catch (Exception e) {
+            assertEquals(0, called.get());
+            assertEquals(1, called1.get());
+        }
+        try {
+            transportInterceptor.interceptHandler("foo/baz/boom", null, false, null);
+        } catch (Exception e) {
+            assertEquals(0, called.get());
+            assertEquals(2, called1.get());
+        }
+    }
+
+    private NetworkModule newNetworkModule(
+        Settings settings,
+        List coreTransportInterceptors,
+        NetworkPlugin... plugins
+    ) {
         return new NetworkModule(
             settings,
             Arrays.asList(plugins),
@@ -322,7 +502,8 @@ private NetworkModule newNetworkModule(Settings settings, NetworkPlugin... plugi
             null,
             new NullDispatcher(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            NoopTracer.INSTANCE
+            NoopTracer.INSTANCE,
+            coreTransportInterceptors
         );
     }
 }
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java
new file mode 100644
index 0000000000000..bac4eaf3fd677
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlServiceTests.java
@@ -0,0 +1,140 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol;
+
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.controllers.AdmissionController;
+import org.opensearch.ratelimitting.admissioncontrol.controllers.CPUBasedAdmissionController;
+import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+import org.opensearch.ratelimitting.admissioncontrol.settings.CPUBasedAdmissionControllerSettings;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.List;
+
+public class AdmissionControlServiceTests extends OpenSearchTestCase {
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+    private AdmissionControlService admissionControlService;
+    private String action = "";
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool("admission_controller_settings_test");
+        clusterService = new ClusterService(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool
+        );
+        action = "indexing";
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdownNow();
+    }
+
+    public void testWhenAdmissionControllerRegistered() {
+        admissionControlService = new AdmissionControlService(Settings.EMPTY, clusterService.getClusterSettings(), threadPool);
+        assertEquals(admissionControlService.getAdmissionControllers().size(), 1);
+    }
+
+    public void testRegisterInvalidAdmissionController() {
+        String test = "TEST";
+        admissionControlService = new AdmissionControlService(Settings.EMPTY, clusterService.getClusterSettings(), threadPool);
+        assertEquals(admissionControlService.getAdmissionControllers().size(), 1);
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> admissionControlService.registerAdmissionController(test)
+        );
+        assertEquals(ex.getMessage(), "Not Supported AdmissionController : " + test);
+    }
+
+    public void testAdmissionControllerSettings() {
+        admissionControlService = new AdmissionControlService(Settings.EMPTY, clusterService.getClusterSettings(), threadPool);
+        AdmissionControlSettings admissionControlSettings = admissionControlService.admissionControlSettings;
+        List admissionControllerList = admissionControlService.getAdmissionControllers();
+        assertEquals(admissionControllerList.size(), 1);
+        CPUBasedAdmissionController cpuBasedAdmissionController = (CPUBasedAdmissionController) admissionControlService
+            .getAdmissionController(CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER);
+        assertEquals(
+            admissionControlSettings.isTransportLayerAdmissionControlEnabled(),
+            cpuBasedAdmissionController.isEnabledForTransportLayer(
+                cpuBasedAdmissionController.settings.getTransportLayerAdmissionControllerMode()
+            )
+        );
+
+        Settings settings = Settings.builder()
+            .put(AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE.getKey(), AdmissionControlMode.DISABLED.getMode())
+            .build();
+        clusterService.getClusterSettings().applySettings(settings);
+        assertEquals(
+            admissionControlSettings.isTransportLayerAdmissionControlEnabled(),
+            cpuBasedAdmissionController.isEnabledForTransportLayer(
+                cpuBasedAdmissionController.settings.getTransportLayerAdmissionControllerMode()
+            )
+        );
+        assertFalse(admissionControlSettings.isTransportLayerAdmissionControlEnabled());
+
+        Settings newSettings = Settings.builder()
+            .put(settings)
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.ENFORCED.getMode()
+            )
+            .build();
+        clusterService.getClusterSettings().applySettings(newSettings);
+        assertFalse(admissionControlSettings.isTransportLayerAdmissionControlEnabled());
+        assertTrue(
+            cpuBasedAdmissionController.isEnabledForTransportLayer(
+                cpuBasedAdmissionController.settings.getTransportLayerAdmissionControllerMode()
+            )
+        );
+    }
+
+    public void testApplyAdmissionControllerDisabled() {
+        this.action = "indices:data/write/bulk[s][p]";
+        admissionControlService = new AdmissionControlService(Settings.EMPTY, clusterService.getClusterSettings(), threadPool);
+        admissionControlService.applyTransportAdmissionControl(this.action);
+        List admissionControllerList = admissionControlService.getAdmissionControllers();
+        admissionControllerList.forEach(admissionController -> { assertEquals(admissionController.getRejectionCount(), 0); });
+    }
+
+    public void testApplyAdmissionControllerEnabled() {
+        this.action = "indices:data/write/bulk[s][p]";
+        admissionControlService = new AdmissionControlService(Settings.EMPTY, clusterService.getClusterSettings(), threadPool);
+        admissionControlService.applyTransportAdmissionControl(this.action);
+        assertEquals(
+            admissionControlService.getAdmissionController(CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER)
+                .getRejectionCount(),
+            0
+        );
+
+        Settings settings = Settings.builder()
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.MONITOR.getMode()
+            )
+            .build();
+        clusterService.getClusterSettings().applySettings(settings);
+        admissionControlService.applyTransportAdmissionControl(this.action);
+        List admissionControllerList = admissionControlService.getAdmissionControllers();
+        assertEquals(admissionControllerList.size(), 1);
+        assertEquals(
+            admissionControlService.getAdmissionController(CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER)
+                .getRejectionCount(),
+            1
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java
new file mode 100644
index 0000000000000..c11ee1cc608f6
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionControlSettingsTests.java
@@ -0,0 +1,103 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol;
+
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.List;
+import java.util.Set;
+
+public class AdmissionControlSettingsTests extends OpenSearchTestCase {
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool("admission_controller_settings_test");
+        clusterService = new ClusterService(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool
+        );
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdownNow();
+    }
+
+    public void testSettingsExists() {
+        Set> settings = ClusterSettings.BUILT_IN_CLUSTER_SETTINGS;
+        assertTrue(
+            "All the admission controller settings should be supported built in settings",
+            settings.containsAll(List.of(AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE))
+        );
+    }
+
+    public void testDefaultSettings() {
+        AdmissionControlSettings admissionControlSettings = new AdmissionControlSettings(
+            clusterService.getClusterSettings(),
+            Settings.EMPTY
+        );
+
+        assertFalse(admissionControlSettings.isTransportLayerAdmissionControlEnabled());
+        assertFalse(admissionControlSettings.isTransportLayerAdmissionControlEnforced());
+        assertEquals(admissionControlSettings.getAdmissionControlTransportLayerMode().getMode(), AdmissionControlSettings.Defaults.MODE);
+    }
+
+    public void testGetConfiguredSettings() {
+        Settings settings = Settings.builder()
+            .put(AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE.getKey(), AdmissionControlMode.ENFORCED.getMode())
+            .build();
+
+        AdmissionControlSettings admissionControlSettings = new AdmissionControlSettings(clusterService.getClusterSettings(), settings);
+
+        assertTrue(admissionControlSettings.isTransportLayerAdmissionControlEnabled());
+        assertTrue(admissionControlSettings.isTransportLayerAdmissionControlEnforced());
+    }
+
+    public void testUpdateAfterGetDefaultSettings() {
+        AdmissionControlSettings admissionControlSettings = new AdmissionControlSettings(
+            clusterService.getClusterSettings(),
+            Settings.EMPTY
+        );
+        Settings settings = Settings.builder()
+            .put(AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE.getKey(), AdmissionControlMode.MONITOR.getMode())
+            .build();
+        clusterService.getClusterSettings().applySettings(settings);
+        assertTrue(admissionControlSettings.isTransportLayerAdmissionControlEnabled());
+        assertFalse(admissionControlSettings.isTransportLayerAdmissionControlEnforced());
+    }
+
+    public void testUpdateAfterGetConfiguredSettings() {
+        Settings settings = Settings.builder()
+            .put(AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE.getKey(), AdmissionControlMode.MONITOR.getMode())
+            .build();
+
+        AdmissionControlSettings admissionControlSettings = new AdmissionControlSettings(clusterService.getClusterSettings(), settings);
+
+        Settings newSettings = Settings.builder()
+            .put(AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE.getKey(), AdmissionControlMode.ENFORCED.getMode())
+            .build();
+
+        clusterService.getClusterSettings().applySettings(newSettings);
+
+        assertTrue(admissionControlSettings.isTransportLayerAdmissionControlEnabled());
+        assertTrue(admissionControlSettings.isTransportLayerAdmissionControlEnforced());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionControllerTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionControllerTests.java
new file mode 100644
index 0000000000000..af6ec0749e709
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/controllers/CPUBasedAdmissionControllerTests.java
@@ -0,0 +1,109 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.controllers;
+
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+import org.opensearch.ratelimitting.admissioncontrol.settings.CPUBasedAdmissionControllerSettings;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+
+public class CPUBasedAdmissionControllerTests extends OpenSearchTestCase {
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+    CPUBasedAdmissionController admissionController = null;
+
+    String action = "TEST_ACTION";
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool("admission_controller_settings_test");
+        clusterService = new ClusterService(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool
+        );
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdownNow();
+    }
+
+    public void testCheckDefaultParameters() {
+        admissionController = new CPUBasedAdmissionController(
+            CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER,
+            Settings.EMPTY,
+            clusterService.getClusterSettings()
+        );
+        assertEquals(admissionController.getName(), CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER);
+        assertEquals(admissionController.getRejectionCount(), 0);
+        assertEquals(admissionController.settings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.DISABLED);
+        assertFalse(
+            admissionController.isEnabledForTransportLayer(admissionController.settings.getTransportLayerAdmissionControllerMode())
+        );
+    }
+
+    public void testCheckUpdateSettings() {
+        admissionController = new CPUBasedAdmissionController(
+            CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER,
+            Settings.EMPTY,
+            clusterService.getClusterSettings()
+        );
+        Settings settings = Settings.builder()
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.ENFORCED.getMode()
+            )
+            .build();
+        clusterService.getClusterSettings().applySettings(settings);
+
+        assertEquals(admissionController.getName(), CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER);
+        assertEquals(admissionController.getRejectionCount(), 0);
+        assertEquals(admissionController.settings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.ENFORCED);
+        assertTrue(admissionController.isEnabledForTransportLayer(admissionController.settings.getTransportLayerAdmissionControllerMode()));
+    }
+
+    public void testApplyControllerWithDefaultSettings() {
+        admissionController = new CPUBasedAdmissionController(
+            CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER,
+            Settings.EMPTY,
+            clusterService.getClusterSettings()
+        );
+        assertEquals(admissionController.getRejectionCount(), 0);
+        assertEquals(admissionController.settings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.DISABLED);
+        action = "indices:data/write/bulk[s][p]";
+        admissionController.apply(action);
+        assertEquals(admissionController.getRejectionCount(), 0);
+    }
+
+    public void testApplyControllerWhenSettingsEnabled() {
+        Settings settings = Settings.builder()
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.ENFORCED.getMode()
+            )
+            .build();
+        admissionController = new CPUBasedAdmissionController(
+            CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER,
+            settings,
+            clusterService.getClusterSettings()
+        );
+        assertTrue(admissionController.isEnabledForTransportLayer(admissionController.settings.getTransportLayerAdmissionControllerMode()));
+        assertEquals(admissionController.getRejectionCount(), 0);
+        action = "indices:data/write/bulk[s][p]";
+        admissionController.apply(action);
+        assertEquals(admissionController.getRejectionCount(), 1);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlModeTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlModeTests.java
new file mode 100644
index 0000000000000..98c0f3c7cf24c
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/AdmissionControlModeTests.java
@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.enums;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+public class AdmissionControlModeTests extends OpenSearchTestCase {
+
+    public void testValidActionType() {
+        assertEquals(AdmissionControlMode.DISABLED.getMode(), "disabled");
+        assertEquals(AdmissionControlMode.ENFORCED.getMode(), "enforced");
+        assertEquals(AdmissionControlMode.MONITOR.getMode(), "monitor_only");
+        assertEquals(AdmissionControlMode.fromName("disabled"), AdmissionControlMode.DISABLED);
+        assertEquals(AdmissionControlMode.fromName("enforced"), AdmissionControlMode.ENFORCED);
+        assertEquals(AdmissionControlMode.fromName("monitor_only"), AdmissionControlMode.MONITOR);
+    }
+
+    public void testInValidActionType() {
+        String name = "TEST";
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> AdmissionControlMode.fromName(name));
+        assertEquals(ex.getMessage(), "Invalid AdmissionControlMode: " + name);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionTypeTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionTypeTests.java
new file mode 100644
index 0000000000000..02f582c26f54e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/enums/TransportActionTypeTests.java
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.enums;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+public class TransportActionTypeTests extends OpenSearchTestCase {
+
+    public void testValidActionType() {
+        assertEquals(TransportActionType.SEARCH.getType(), "search");
+        assertEquals(TransportActionType.INDEXING.getType(), "indexing");
+        assertEquals(TransportActionType.fromName("search"), TransportActionType.SEARCH);
+        assertEquals(TransportActionType.fromName("indexing"), TransportActionType.INDEXING);
+    }
+
+    public void testInValidActionType() {
+        String name = "test";
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> TransportActionType.fromName(name));
+        assertEquals(ex.getMessage(), "Not Supported TransportAction Type: " + name);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControlSettingsTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControlSettingsTests.java
new file mode 100644
index 0000000000000..43103926a69a2
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/settings/CPUBasedAdmissionControlSettingsTests.java
@@ -0,0 +1,153 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.settings;
+
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlMode;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.Arrays;
+import java.util.Set;
+
+public class CPUBasedAdmissionControlSettingsTests extends OpenSearchTestCase {
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool("admission_controller_settings_test");
+        clusterService = new ClusterService(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool
+        );
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdownNow();
+    }
+
+    public void testSettingsExists() {
+        Set> settings = ClusterSettings.BUILT_IN_CLUSTER_SETTINGS;
+        assertTrue(
+            "All the cpu based admission controller settings should be supported built in settings",
+            settings.containsAll(
+                Arrays.asList(
+                    CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
+                    CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
+                    CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT
+                )
+            )
+        );
+    }
+
+    public void testDefaultSettings() {
+        CPUBasedAdmissionControllerSettings cpuBasedAdmissionControllerSettings = new CPUBasedAdmissionControllerSettings(
+            clusterService.getClusterSettings(),
+            Settings.EMPTY
+        );
+        long percent = 95;
+        assertEquals(cpuBasedAdmissionControllerSettings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.DISABLED);
+        assertEquals(cpuBasedAdmissionControllerSettings.getIndexingCPULimit().longValue(), percent);
+        assertEquals(cpuBasedAdmissionControllerSettings.getSearchCPULimit().longValue(), percent);
+        assertEquals(cpuBasedAdmissionControllerSettings.getTransportActionsList(), Arrays.asList("indexing", "search"));
+    }
+
+    public void testGetConfiguredSettings() {
+        long percent = 95;
+        long indexingPercent = 85;
+        Settings settings = Settings.builder()
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.ENFORCED.getMode()
+            )
+            .put(CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT.getKey(), indexingPercent)
+            .build();
+
+        CPUBasedAdmissionControllerSettings cpuBasedAdmissionControllerSettings = new CPUBasedAdmissionControllerSettings(
+            clusterService.getClusterSettings(),
+            settings
+        );
+        assertEquals(cpuBasedAdmissionControllerSettings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.ENFORCED);
+        assertEquals(cpuBasedAdmissionControllerSettings.getSearchCPULimit().longValue(), percent);
+        assertEquals(cpuBasedAdmissionControllerSettings.getIndexingCPULimit().longValue(), indexingPercent);
+    }
+
+    public void testUpdateAfterGetDefaultSettings() {
+        long percent = 95;
+        long searchPercent = 80;
+        CPUBasedAdmissionControllerSettings cpuBasedAdmissionControllerSettings = new CPUBasedAdmissionControllerSettings(
+            clusterService.getClusterSettings(),
+            Settings.EMPTY
+        );
+        Settings settings = Settings.builder()
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.ENFORCED.getMode()
+            )
+            .put(CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT.getKey(), searchPercent)
+            .build();
+        clusterService.getClusterSettings().applySettings(settings);
+        assertEquals(cpuBasedAdmissionControllerSettings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.ENFORCED);
+        assertEquals(cpuBasedAdmissionControllerSettings.getSearchCPULimit().longValue(), searchPercent);
+        assertEquals(cpuBasedAdmissionControllerSettings.getIndexingCPULimit().longValue(), percent);
+    }
+
+    public void testUpdateAfterGetConfiguredSettings() {
+        long percent = 95;
+        long indexingPercent = 85;
+        long searchPercent = 80;
+        Settings settings = Settings.builder()
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.ENFORCED.getMode()
+            )
+            .put(CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT.getKey(), searchPercent)
+            .build();
+
+        CPUBasedAdmissionControllerSettings cpuBasedAdmissionControllerSettings = new CPUBasedAdmissionControllerSettings(
+            clusterService.getClusterSettings(),
+            settings
+        );
+        assertEquals(cpuBasedAdmissionControllerSettings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.ENFORCED);
+        assertEquals(cpuBasedAdmissionControllerSettings.getSearchCPULimit().longValue(), searchPercent);
+        assertEquals(cpuBasedAdmissionControllerSettings.getIndexingCPULimit().longValue(), percent);
+
+        Settings updatedSettings = Settings.builder()
+            .put(
+                CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE.getKey(),
+                AdmissionControlMode.MONITOR.getMode()
+            )
+            .put(CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT.getKey(), indexingPercent)
+            .build();
+        clusterService.getClusterSettings().applySettings(updatedSettings);
+        assertEquals(cpuBasedAdmissionControllerSettings.getTransportLayerAdmissionControllerMode(), AdmissionControlMode.MONITOR);
+        assertEquals(cpuBasedAdmissionControllerSettings.getSearchCPULimit().longValue(), searchPercent);
+        assertEquals(cpuBasedAdmissionControllerSettings.getIndexingCPULimit().longValue(), indexingPercent);
+
+        searchPercent = 70;
+
+        updatedSettings = Settings.builder()
+            .put(updatedSettings)
+            .put(CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT.getKey(), searchPercent)
+            .build();
+        clusterService.getClusterSettings().applySettings(updatedSettings);
+
+        assertEquals(cpuBasedAdmissionControllerSettings.getSearchCPULimit().longValue(), searchPercent);
+        assertEquals(cpuBasedAdmissionControllerSettings.getIndexingCPULimit().longValue(), indexingPercent);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandlerTests.java b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandlerTests.java
new file mode 100644
index 0000000000000..03d4819a94045
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ratelimitting/admissioncontrol/transport/AdmissionControlTransportHandlerTests.java
@@ -0,0 +1,92 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ratelimitting.admissioncontrol.transport;
+
+import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
+import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
+import org.opensearch.tasks.Task;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.transport.TransportChannel;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportRequestHandler;
+
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+
+public class AdmissionControlTransportHandlerTests extends OpenSearchTestCase {
+    AdmissionControlTransportHandler admissionControlTransportHandler;
+
+    public void testHandlerInvoked() throws Exception {
+        String action = "TEST";
+        InterceptingRequestHandler handler = new InterceptingRequestHandler<>(action);
+        admissionControlTransportHandler = new AdmissionControlTransportHandler(
+            action,
+            handler,
+            mock(AdmissionControlService.class),
+            false
+        );
+        admissionControlTransportHandler.messageReceived(mock(TransportRequest.class), mock(TransportChannel.class), mock(Task.class));
+        assertEquals(1, handler.count);
+    }
+
+    public void testHandlerInvokedRejectedException() throws Exception {
+        String action = "TEST";
+        AdmissionControlService admissionControlService = mock(AdmissionControlService.class);
+        doThrow(new OpenSearchRejectedExecutionException()).when(admissionControlService).applyTransportAdmissionControl(action);
+        InterceptingRequestHandler handler = new InterceptingRequestHandler<>(action);
+        admissionControlTransportHandler = new AdmissionControlTransportHandler(
+            action,
+            handler,
+            admissionControlService,
+            false
+        );
+        try {
+            admissionControlTransportHandler.messageReceived(mock(TransportRequest.class), mock(TransportChannel.class), mock(Task.class));
+        } catch (OpenSearchRejectedExecutionException exception) {
+            assertEquals(0, handler.count);
+            handler.messageReceived(mock(TransportRequest.class), mock(TransportChannel.class), mock(Task.class));
+        }
+        assertEquals(1, handler.count);
+    }
+
+    public void testHandlerInvokedRandomException() throws Exception {
+        String action = "TEST";
+        AdmissionControlService admissionControlService = mock(AdmissionControlService.class);
+        doThrow(new NullPointerException()).when(admissionControlService).applyTransportAdmissionControl(action);
+        InterceptingRequestHandler handler = new InterceptingRequestHandler<>(action);
+        admissionControlTransportHandler = new AdmissionControlTransportHandler(
+            action,
+            handler,
+            admissionControlService,
+            false
+        );
+        try {
+            admissionControlTransportHandler.messageReceived(mock(TransportRequest.class), mock(TransportChannel.class), mock(Task.class));
+        } catch (Exception exception) {
+            assertEquals(0, handler.count);
+            handler.messageReceived(mock(TransportRequest.class), mock(TransportChannel.class), mock(Task.class));
+        }
+        assertEquals(1, handler.count);
+    }
+
+    private class InterceptingRequestHandler implements TransportRequestHandler {
+        private final String action;
+        public int count;
+
+        public InterceptingRequestHandler(String action) {
+            this.action = action;
+            this.count = 0;
+        }
+
+        @Override
+        public void messageReceived(T request, TransportChannel channel, Task task) throws Exception {
+            this.count = this.count + 1;
+        }
+    }
+}

From 14d4a6389bd5a7612c14e2e6fcb5a39822af9ee0 Mon Sep 17 00:00:00 2001
From: Ashish 
Date: Sat, 21 Oct 2023 18:42:52 +0530
Subject: [PATCH 209/388] [Remote Store] Use time elapsed since last successful
 local refresh for refresh lag (#10803)

* [Remote Store] Use time elapsed since last successful local refresh for time lag

Signed-off-by: Ashish Singh 

* Incorporate PR review comments

Signed-off-by: Ashish Singh 

---------

Signed-off-by: Ashish Singh 
---
 ...emoteStoreBackpressureAndResiliencyIT.java |  6 ++-
 .../remote/RemoteSegmentTransferTracker.java  | 53 ++++++++++++-------
 .../remote/RemoteStorePressureService.java    |  1 -
 .../RemoteSegmentTransferTrackerTests.java    | 39 +++++++++-----
 .../RemoteStorePressureServiceTests.java      | 32 ++++++++---
 5 files changed, 86 insertions(+), 45 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
index 98586b60dcc69..f19c9db7874db 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
@@ -56,7 +56,7 @@ public void testWritesRejectedDueToBytesLagBreach() throws Exception {
     public void testWritesRejectedDueToTimeLagBreach() throws Exception {
         // Initially indexing happens with doc size of 1KB, then all remote store interactions start failing. Now, the
         // indexing happens with doc size of 1 byte leading to time lag limit getting exceeded and leading to rejections.
-        validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 20, ByteSizeUnit.BYTES.toIntBytes(1), 15, "time_lag");
+        validateBackpressure(ByteSizeUnit.KB.toIntBytes(1), 20, ByteSizeUnit.BYTES.toIntBytes(1), 3, "time_lag");
     }
 
     private void validateBackpressure(
@@ -133,11 +133,13 @@ private RemoteSegmentTransferTracker.Stats stats() {
         return matches.get(0).getSegmentStats();
     }
 
-    private void indexDocAndRefresh(BytesReference source, int iterations) {
+    private void indexDocAndRefresh(BytesReference source, int iterations) throws InterruptedException {
         for (int i = 0; i < iterations; i++) {
             client().prepareIndex(INDEX_NAME).setSource(source, MediaTypeRegistry.JSON).get();
             refresh(INDEX_NAME);
         }
+        Thread.sleep(250);
+        client().prepareIndex(INDEX_NAME).setSource(source, MediaTypeRegistry.JSON).get();
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
index 2a703f17aa953..fb65d9ef83be2 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
@@ -66,6 +67,12 @@ public class RemoteSegmentTransferTracker extends RemoteTransferTracker {
      */
     private volatile long remoteRefreshTimeMs;
 
+    /**
+     * This is the time of first local refresh after the last successful remote refresh. When the remote store is in
+     * sync with local refresh, this will be reset to -1.
+     */
+    private volatile long remoteRefreshStartTimeMs = -1;
+
     /**
      * The refresh time(clock) of most recent remote refresh.
      */
@@ -76,11 +83,6 @@ public class RemoteSegmentTransferTracker extends RemoteTransferTracker {
      */
     private volatile long refreshSeqNoLag;
 
-    /**
-     * Keeps the time (ms) lag computed so that we do not compute it for every request.
-     */
-    private volatile long timeMsLag;
-
     /**
      * Keeps track of the total bytes of segment files which were uploaded to remote store during last successful remote refresh
      */
@@ -132,14 +134,19 @@ public RemoteSegmentTransferTracker(
         logger = Loggers.getLogger(getClass(), shardId);
         // Both the local refresh time and remote refresh time are set with current time to give consistent view of time lag when it arises.
         long currentClockTimeMs = System.currentTimeMillis();
-        long currentTimeMs = System.nanoTime() / 1_000_000L;
+        long currentTimeMs = currentTimeMsUsingSystemNanos();
         localRefreshTimeMs = currentTimeMs;
         remoteRefreshTimeMs = currentTimeMs;
+        remoteRefreshStartTimeMs = currentTimeMs;
         localRefreshClockTimeMs = currentClockTimeMs;
         remoteRefreshClockTimeMs = currentClockTimeMs;
         this.directoryFileTransferTracker = directoryFileTransferTracker;
     }
 
+    public static long currentTimeMsUsingSystemNanos() {
+        return TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
+    }
+
     @Override
     public void incrementTotalUploadsFailed() {
         super.incrementTotalUploadsFailed();
@@ -180,19 +187,22 @@ public long getLocalRefreshClockTimeMs() {
      */
     public void updateLocalRefreshTimeAndSeqNo() {
         updateLocalRefreshClockTimeMs(System.currentTimeMillis());
-        updateLocalRefreshTimeMs(System.nanoTime() / 1_000_000L);
+        updateLocalRefreshTimeMs(currentTimeMsUsingSystemNanos());
         updateLocalRefreshSeqNo(getLocalRefreshSeqNo() + 1);
     }
 
     // Visible for testing
-    void updateLocalRefreshTimeMs(long localRefreshTimeMs) {
+    synchronized void updateLocalRefreshTimeMs(long localRefreshTimeMs) {
         assert localRefreshTimeMs >= this.localRefreshTimeMs : "newLocalRefreshTimeMs="
             + localRefreshTimeMs
             + " < "
             + "currentLocalRefreshTimeMs="
             + this.localRefreshTimeMs;
+        boolean isRemoteInSyncBeforeLocalRefresh = this.localRefreshTimeMs == this.remoteRefreshTimeMs;
         this.localRefreshTimeMs = localRefreshTimeMs;
-        computeTimeMsLag();
+        if (isRemoteInSyncBeforeLocalRefresh) {
+            this.remoteRefreshStartTimeMs = localRefreshTimeMs;
+        }
     }
 
     private void updateLocalRefreshClockTimeMs(long localRefreshClockTimeMs) {
@@ -221,14 +231,18 @@ long getRemoteRefreshClockTimeMs() {
         return remoteRefreshClockTimeMs;
     }
 
-    public void updateRemoteRefreshTimeMs(long remoteRefreshTimeMs) {
-        assert remoteRefreshTimeMs >= this.remoteRefreshTimeMs : "newRemoteRefreshTimeMs="
-            + remoteRefreshTimeMs
+    public synchronized void updateRemoteRefreshTimeMs(long refreshTimeMs) {
+        assert refreshTimeMs >= this.remoteRefreshTimeMs : "newRemoteRefreshTimeMs="
+            + refreshTimeMs
             + " < "
             + "currentRemoteRefreshTimeMs="
             + this.remoteRefreshTimeMs;
-        this.remoteRefreshTimeMs = remoteRefreshTimeMs;
-        computeTimeMsLag();
+        this.remoteRefreshTimeMs = refreshTimeMs;
+        // When multiple refreshes have failed, there is a possibility that retry is ongoing while another refresh gets
+        // triggered. After the segments have been uploaded and before the below code runs, the updateLocalRefreshTimeAndSeqNo
+        // method is triggered, which will update the local localRefreshTimeMs. Now, the lag would basically become the
+        // time since the last refresh happened locally.
+        this.remoteRefreshStartTimeMs = refreshTimeMs == this.localRefreshTimeMs ? -1 : this.localRefreshTimeMs;
     }
 
     public void updateRemoteRefreshClockTimeMs(long remoteRefreshClockTimeMs) {
@@ -243,12 +257,11 @@ public long getRefreshSeqNoLag() {
         return refreshSeqNoLag;
     }
 
-    private void computeTimeMsLag() {
-        timeMsLag = localRefreshTimeMs - remoteRefreshTimeMs;
-    }
-
     public long getTimeMsLag() {
-        return timeMsLag;
+        if (remoteRefreshTimeMs == localRefreshTimeMs) {
+            return 0;
+        }
+        return currentTimeMsUsingSystemNanos() - remoteRefreshStartTimeMs;
     }
 
     public long getBytesLag() {
@@ -354,7 +367,7 @@ public RemoteSegmentTransferTracker.Stats stats() {
             shardId,
             localRefreshClockTimeMs,
             remoteRefreshClockTimeMs,
-            timeMsLag,
+            getTimeMsLag(),
             localRefreshSeqNo,
             remoteRefreshSeqNo,
             uploadBytesStarted.get(),
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java
index 2920b33921869..33cd40f802d43 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java
@@ -180,7 +180,6 @@ public boolean validate(RemoteSegmentTransferTracker pressureTracker, ShardId sh
                 return true;
             }
             if (pressureTracker.isUploadTimeMovingAverageReady() == false) {
-                logger.trace("upload time moving average is not ready");
                 return true;
             }
             long timeLag = pressureTracker.getTimeMsLag();
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
index 0bf00f9e48137..c87cdfcc8f1a1 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java
@@ -23,6 +23,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.opensearch.index.remote.RemoteSegmentTransferTracker.currentTimeMsUsingSystemNanos;
+
 public class RemoteSegmentTransferTrackerTests extends OpenSearchTestCase {
     private RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory;
     private ClusterService clusterService;
@@ -92,7 +94,7 @@ public void testUpdateLocalRefreshTimeMs() {
             directoryFileTransferTracker,
             remoteStoreStatsTrackerFactory.getMovingAverageWindowSize()
         );
-        long refreshTimeMs = System.nanoTime() / 1_000_000L + randomIntBetween(10, 100);
+        long refreshTimeMs = currentTimeMsUsingSystemNanos() + randomIntBetween(10, 100);
         transferTracker.updateLocalRefreshTimeMs(refreshTimeMs);
         assertEquals(refreshTimeMs, transferTracker.getLocalRefreshTimeMs());
     }
@@ -103,7 +105,7 @@ public void testUpdateRemoteRefreshTimeMs() {
             directoryFileTransferTracker,
             remoteStoreStatsTrackerFactory.getMovingAverageWindowSize()
         );
-        long refreshTimeMs = System.nanoTime() / 1_000_000 + randomIntBetween(10, 100);
+        long refreshTimeMs = currentTimeMsUsingSystemNanos() + randomIntBetween(10, 100);
         transferTracker.updateRemoteRefreshTimeMs(refreshTimeMs);
         assertEquals(refreshTimeMs, transferTracker.getRemoteRefreshTimeMs());
     }
@@ -133,20 +135,29 @@ public void testComputeSeqNoLagOnUpdate() {
         assertEquals(localRefreshSeqNo - remoteRefreshSeqNo, transferTracker.getRefreshSeqNoLag());
     }
 
-    public void testComputeTimeLagOnUpdate() {
+    public void testComputeTimeLagOnUpdate() throws InterruptedException {
         transferTracker = new RemoteSegmentTransferTracker(
             shardId,
             directoryFileTransferTracker,
             remoteStoreStatsTrackerFactory.getMovingAverageWindowSize()
         );
-        long currentLocalRefreshTimeMs = transferTracker.getLocalRefreshTimeMs();
-        long currentTimeMs = System.nanoTime() / 1_000_000L;
-        long localRefreshTimeMs = currentTimeMs + randomIntBetween(100, 500);
-        long remoteRefreshTimeMs = currentTimeMs + randomIntBetween(50, 99);
-        transferTracker.updateLocalRefreshTimeMs(localRefreshTimeMs);
-        assertEquals(localRefreshTimeMs - currentLocalRefreshTimeMs, transferTracker.getTimeMsLag());
-        transferTracker.updateRemoteRefreshTimeMs(remoteRefreshTimeMs);
-        assertEquals(localRefreshTimeMs - remoteRefreshTimeMs, transferTracker.getTimeMsLag());
+
+        // No lag if there is a remote upload corresponding to a local refresh
+        assertEquals(0, transferTracker.getTimeMsLag());
+
+        // Set a local refresh time that is higher than remote refresh time
+        Thread.sleep(1);
+        transferTracker.updateLocalRefreshTimeMs(currentTimeMsUsingSystemNanos());
+
+        // Sleep for 100ms and then the lag should be within 100ms +/- 20ms
+        Thread.sleep(100);
+        assertTrue(Math.abs(transferTracker.getTimeMsLag() - 100) <= 20);
+
+        transferTracker.updateRemoteRefreshTimeMs(transferTracker.getLocalRefreshTimeMs());
+        transferTracker.updateLocalRefreshTimeMs(currentTimeMsUsingSystemNanos());
+        long random = randomIntBetween(50, 200);
+        Thread.sleep(random);
+        assertTrue(Math.abs(transferTracker.getTimeMsLag() - random) <= 20);
     }
 
     public void testAddUploadBytesStarted() {
@@ -519,7 +530,7 @@ public void testStatsObjectCreation() {
         transferTracker = constructTracker();
         RemoteSegmentTransferTracker.Stats transferTrackerStats = transferTracker.stats();
         assertEquals(transferTracker.getShardId(), transferTrackerStats.shardId);
-        assertEquals(transferTracker.getTimeMsLag(), (int) transferTrackerStats.refreshTimeLagMs);
+        assertTrue(Math.abs(transferTracker.getTimeMsLag() - transferTrackerStats.refreshTimeLagMs) <= 20);
         assertEquals(transferTracker.getLocalRefreshSeqNo(), (int) transferTrackerStats.localRefreshNumber);
         assertEquals(transferTracker.getRemoteRefreshSeqNo(), (int) transferTrackerStats.remoteRefreshNumber);
         assertEquals(transferTracker.getBytesLag(), (int) transferTrackerStats.bytesLag);
@@ -591,9 +602,9 @@ private RemoteSegmentTransferTracker constructTracker() {
         );
         transferTracker.incrementTotalUploadsStarted();
         transferTracker.incrementTotalUploadsFailed();
-        transferTracker.updateUploadTimeMovingAverage(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100));
+        transferTracker.updateUploadTimeMovingAverage(currentTimeMsUsingSystemNanos() + randomIntBetween(10, 100));
         transferTracker.updateUploadBytesMovingAverage(99);
-        transferTracker.updateRemoteRefreshTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100));
+        transferTracker.updateRemoteRefreshTimeMs(currentTimeMsUsingSystemNanos() + randomIntBetween(10, 100));
         transferTracker.incrementRejectionCount();
         transferTracker.getDirectoryFileTransferTracker().addTransferredBytesStarted(10);
         transferTracker.getDirectoryFileTransferTracker().addTransferredBytesSucceeded(10, System.currentTimeMillis());
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java
index de610083f3327..cb77174e612fd 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePressureServiceTests.java
@@ -21,8 +21,11 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.IntStream;
 
+import static org.opensearch.index.remote.RemoteSegmentTransferTracker.currentTimeMsUsingSystemNanos;
 import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexShard;
 
 public class RemoteStorePressureServiceTests extends OpenSearchTestCase {
@@ -68,7 +71,7 @@ public void testIsSegmentsUploadBackpressureEnabled() {
         assertTrue(pressureService.isSegmentsUploadBackpressureEnabled());
     }
 
-    public void testValidateSegmentUploadLag() {
+    public void testValidateSegmentUploadLag() throws InterruptedException {
         // Create the pressure tracker
         IndexShard indexShard = createIndexShard(shardId, true);
         remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, Settings.EMPTY);
@@ -86,14 +89,27 @@ public void testValidateSegmentUploadLag() {
             sum.addAndGet(i);
         });
         double avg = (double) sum.get() / 20;
-        long currentMs = System.nanoTime() / 1_000_000;
-        pressureTracker.updateLocalRefreshTimeMs((long) (currentMs + 12 * avg));
-        pressureTracker.updateRemoteRefreshTimeMs(currentMs);
-        Exception e = assertThrows(OpenSearchRejectedExecutionException.class, () -> pressureService.validateSegmentsUploadLag(shardId));
-        assertTrue(e.getMessage().contains("due to remote segments lagging behind local segments"));
-        assertTrue(e.getMessage().contains("time_lag:114 ms dynamic_time_lag_threshold:95.0 ms"));
 
-        pressureTracker.updateRemoteRefreshTimeMs((long) (currentMs + 2 * avg));
+        // We run this to ensure that the local and remote refresh time are not same anymore
+        while (pressureTracker.getLocalRefreshTimeMs() == currentTimeMsUsingSystemNanos()) {
+            Thread.sleep(10);
+        }
+        long localRefreshTimeMs = currentTimeMsUsingSystemNanos();
+        pressureTracker.updateLocalRefreshTimeMs(localRefreshTimeMs);
+
+        while (currentTimeMsUsingSystemNanos() - localRefreshTimeMs <= 20 * avg) {
+            Thread.sleep((long) (4 * avg));
+        }
+        Exception e = assertThrows(OpenSearchRejectedExecutionException.class, () -> pressureService.validateSegmentsUploadLag(shardId));
+        String regex = "^rejected execution on primary shard:\\[index]\\[0] due to remote segments lagging behind "
+            + "local segments.time_lag:[0-9]{2,3} ms dynamic_time_lag_threshold:95\\.0 ms$";
+        Pattern pattern = Pattern.compile(regex);
+        Matcher matcher = pattern.matcher(e.getMessage());
+        assertTrue(matcher.matches());
+
+        pressureTracker.updateRemoteRefreshTimeMs(pressureTracker.getLocalRefreshTimeMs());
+        pressureTracker.updateLocalRefreshTimeMs(currentTimeMsUsingSystemNanos());
+        Thread.sleep((long) (2 * avg));
         pressureService.validateSegmentsUploadLag(shardId);
 
         // 2. bytes lag more than dynamic threshold

From 911afc4002b3a9faa86e109871332af0caaa73d0 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Sun, 22 Oct 2023 11:31:31 +0530
Subject: [PATCH 210/388] Remove remote store attributes from DiscoveryNode
 toString() (#10810)

Signed-off-by: Dhwanil Patel 
---
 .../cluster/node/DiscoveryNode.java           |  8 +++++++-
 .../cluster/node/DiscoveryNodeTests.java      | 19 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
index 4e49b25eb5789..0c58aabf95207 100644
--- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java
@@ -553,7 +553,13 @@ public String toString() {
             sb.append('}');
         }
         if (!attributes.isEmpty()) {
-            sb.append(attributes);
+            sb.append(
+                attributes.entrySet()
+                    .stream()
+                    .filter(entry -> !entry.getKey().startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)) // filter remote_store attributes
+                                                                                                         // from logging to reduce noise.
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
+            );
         }
         return sb.toString();
     }
diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java
index 8b61e8f6d724d..c8a6fc76ce820 100644
--- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java
+++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java
@@ -38,13 +38,16 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
 import org.opensearch.test.NodeRoles;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.net.InetAddress;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -81,6 +84,22 @@ public void testRolesAreSorted() {
 
     }
 
+    public void testRemoteStoreRedactionInToString() {
+        final Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES));
+        Map attributes = new HashMap<>();
+        attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo");
+        attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo");
+        final DiscoveryNode node = new DiscoveryNode(
+            "name",
+            "id",
+            new TransportAddress(TransportAddress.META_ADDRESS, 9200),
+            attributes,
+            roles,
+            Version.CURRENT
+        );
+        assertFalse(node.toString().contains(RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX));
+    }
+
     public void testDiscoveryNodeIsCreatedWithHostFromInetAddress() throws Exception {
         InetAddress inetAddress = randomBoolean()
             ? InetAddress.getByName("192.0.2.1")

From 79c6e322f584e72b48793658f6a3456d1eea1fc2 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Sun, 22 Oct 2023 11:54:10 +0530
Subject: [PATCH 211/388] Adds util to compare Metadata for remote state
 (#10661)

* Adds util to compare Metadata for remote state durng restore flow when identifying if 2 clusterUUIDs have same cluster state

Signed-off-by: bansvaru 
---
 .../opensearch/cluster/metadata/Metadata.java |  17 +-
 .../remote/RemoteClusterStateService.java     |  12 +-
 .../cluster/metadata/MetadataTests.java       |  33 ++++
 .../RemoteClusterStateServiceTests.java       | 163 ++++++++++++------
 4 files changed, 168 insertions(+), 57 deletions(-)

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
index 626903877b0c6..70c1d059a1b9e 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
@@ -922,19 +922,26 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
         if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) {
             return false;
         }
-        if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
+        if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
             return false;
         }
-        if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
+        if (!metadata1.clusterUUID.equals(metadata2.clusterUUID)) {
             return false;
         }
-        if (!metadata1.templates.equals(metadata2.templates())) {
+        if (metadata1.clusterUUIDCommitted != metadata2.clusterUUIDCommitted) {
             return false;
         }
-        if (!metadata1.clusterUUID.equals(metadata2.clusterUUID)) {
+        return isGlobalResourcesMetadataEquals(metadata1, metadata2);
+    }
+
+    /**
+     * Compares Metadata entities persisted in Remote Store.
+     */
+    public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) {
+        if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
             return false;
         }
-        if (metadata1.clusterUUIDCommitted != metadata2.clusterUUIDCommitted) {
+        if (!metadata1.templates.equals(metadata2.templates())) {
             return false;
         }
         // Check if any persistent metadata needs to be saved
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 96ce2fc779ea0..13fbda77d8a2d 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -689,10 +689,11 @@ private Map getIndexMetadataMap(
      * @return {@link IndexMetadata}
      */
     private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, UploadedIndexMetadata uploadedIndexMetadata) {
+        BlobContainer blobContainer = indexMetadataContainer(clusterName, clusterUUID, uploadedIndexMetadata.getIndexUUID());
         try {
             String[] splitPath = uploadedIndexMetadata.getUploadedFilename().split("/");
             return INDEX_METADATA_FORMAT.read(
-                indexMetadataContainer(clusterName, clusterUUID, uploadedIndexMetadata.getIndexUUID()),
+                blobContainer,
                 splitPath[splitPath.length - 1],
                 blobStoreRepository.getNamedXContentRegistry()
             );
@@ -882,7 +883,8 @@ private Map trimClusterUUIDs(
                 }
             } else {
                 ClusterMetadataManifest previousManifest = trimmedUUIDs.get(currentManifest.getPreviousClusterUUID());
-                if (isMetadataEqual(currentManifest, previousManifest, clusterName)) {
+                if (isMetadataEqual(currentManifest, previousManifest, clusterName)
+                    && isGlobalMetadataEqual(currentManifest, previousManifest, clusterName)) {
                     trimmedUUIDs.remove(clusterUUID);
                 }
             }
@@ -912,6 +914,12 @@ private boolean isMetadataEqual(ClusterMetadataManifest first, ClusterMetadataMa
         return true;
     }
 
+    private boolean isGlobalMetadataEqual(ClusterMetadataManifest first, ClusterMetadataManifest second, String clusterName) {
+        Metadata secondGlobalMetadata = getGlobalMetadata(clusterName, second.getClusterUUID(), second);
+        Metadata firstGlobalMetadata = getGlobalMetadata(clusterName, first.getClusterUUID(), first);
+        return Metadata.isGlobalResourcesMetadataEquals(firstGlobalMetadata, secondGlobalMetadata);
+    }
+
     private boolean isInvalidClusterUUID(ClusterMetadataManifest manifest) {
         return !manifest.isClusterUUIDCommitted();
     }
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java
index 40eefa6cdbf03..618fcb923bc60 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java
@@ -627,6 +627,39 @@ public void testGlobalStateEqualsCoordinationMetadata() {
         assertFalse(Metadata.isGlobalStateEquals(metadata1, metadata2));
     }
 
+    public void testGlobalResourcesStateEqualsCoordinationMetadata() {
+        CoordinationMetadata coordinationMetadata1 = new CoordinationMetadata(
+            randomNonNegativeLong(),
+            randomVotingConfig(),
+            randomVotingConfig(),
+            randomVotingConfigExclusions()
+        );
+        Metadata metadata1 = Metadata.builder()
+            .coordinationMetadata(coordinationMetadata1)
+            .clusterUUID(randomAlphaOfLength(10))
+            .clusterUUIDCommitted(false)
+            .hashesOfConsistentSettings(Map.of("a", "b"))
+            .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), true).build())
+            .build();
+        CoordinationMetadata coordinationMetadata2 = new CoordinationMetadata(
+            randomNonNegativeLong(),
+            randomVotingConfig(),
+            randomVotingConfig(),
+            randomVotingConfigExclusions()
+        );
+        Metadata metadata2 = Metadata.builder()
+            .coordinationMetadata(coordinationMetadata2)
+            .clusterUUIDCommitted(true)
+            .clusterUUID(randomAlphaOfLength(11))
+            .hashesOfConsistentSettings(Map.of("b", "a"))
+            .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), true).build())
+            .build();
+
+        assertTrue(Metadata.isGlobalStateEquals(metadata1, metadata1));
+        assertFalse(Metadata.isGlobalStateEquals(metadata1, metadata2));
+        assertTrue(Metadata.isGlobalResourcesMetadataEquals(metadata1, metadata2));
+    }
+
     public void testSerializationWithIndexGraveyard() throws IOException {
         final IndexGraveyard graveyard = IndexGraveyardTests.createRandom();
         final Metadata originalMeta = Metadata.builder().indexGraveyard(graveyard).build();
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 5202f31c514ed..55094400581b4 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -9,6 +9,7 @@
 package org.opensearch.gateway.remote;
 
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.CoordinationMetadata;
@@ -27,6 +28,7 @@
 import org.opensearch.common.blobstore.transfer.RemoteTransferContainer;
 import org.opensearch.common.compress.DeflateCompressor;
 import org.opensearch.common.lucene.store.ByteArrayIndexInput;
+import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.ParseField;
@@ -37,6 +39,7 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
 import org.opensearch.index.remote.RemoteStoreUtils;
+import org.opensearch.indices.IndicesModule;
 import org.opensearch.repositories.FilterRepository;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.RepositoryMissingException;
@@ -65,11 +68,14 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 
+import static java.util.stream.Collectors.toList;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION;
@@ -126,11 +132,19 @@ public void setup() {
             .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
             .build();
 
+        NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(
+            Stream.of(
+                NetworkModule.getNamedXContents().stream(),
+                IndicesModule.getNamedXContents().stream(),
+                ClusterModule.getNamedXWriteables().stream()
+            ).flatMap(Function.identity()).collect(toList())
+        );
+
         blobStoreRepository = mock(BlobStoreRepository.class);
         blobStore = mock(BlobStore.class);
         when(blobStoreRepository.blobStore()).thenReturn(blobStore);
         when(repositoriesService.repository("remote_store_repository")).thenReturn(blobStoreRepository);
-        when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(new NamedXContentRegistry(new ArrayList<>()));
+        when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(xContentRegistry);
         remoteClusterStateService = new RemoteClusterStateService(
             "test-node-id",
             repositoriesServiceSupplier,
@@ -887,7 +901,7 @@ public void testGetValidPreviousClusterUUIDWithMultipleChains() throws IOExcepti
             "cluster-uuid3",
             "cluster-uuid1"
         );
-        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers);
+        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, randomBoolean());
 
         remoteClusterStateService.start();
         String previousClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster");
@@ -1040,6 +1054,11 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti
     }
 
     private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException {
+        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false);
+    }
+
+    private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers, boolean differGlobalMetadata)
+        throws IOException {
         final BlobPath blobPath = mock(BlobPath.class);
         when((blobStoreRepository.basePath())).thenReturn(blobPath);
         when(blobPath.add(anyString())).thenReturn(blobPath);
@@ -1061,7 +1080,8 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste
             "cluster-uuid1",
             clusterUUIDsPointers.get("cluster-uuid1"),
             randomAlphaOfLength(10),
-            uploadedIndexMetadataList1
+            uploadedIndexMetadataList1,
+            "test-metadata1"
         );
         Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexMetadata indexMetadata1 = IndexMetadata.builder("index1")
@@ -1074,8 +1094,12 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
+        Metadata metadata1 = Metadata.builder()
+            .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), true).build())
+            .build();
         Map indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2);
-        mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1);
+        mockBlobContainerForGlobalMetadata(blobContainer1, clusterManifest1, metadata1);
+        mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V1);
 
         List uploadedIndexMetadataList2 = List.of(
             new UploadedIndexMetadata("index1", "index-uuid1", "key1"),
@@ -1085,7 +1109,8 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste
             "cluster-uuid2",
             clusterUUIDsPointers.get("cluster-uuid2"),
             randomAlphaOfLength(10),
-            uploadedIndexMetadataList2
+            uploadedIndexMetadataList2,
+            "test-metadata2"
         );
         IndexMetadata indexMetadata3 = IndexMetadata.builder("index1")
             .settings(indexSettings)
@@ -1097,37 +1122,59 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
+        Metadata metadata2 = Metadata.builder()
+            .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), true).build())
+            .build();
         Map indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4);
-        mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2);
+        mockBlobContainerForGlobalMetadata(blobContainer2, clusterManifest2, metadata2);
+        mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V1);
+
+        // differGlobalMetadata controls which one of IndexMetadata or Metadata object would be different
+        // when comparing cluster-uuid3 and cluster-uuid1 state.
+        // if set true, only Metadata will differ b/w cluster uuid1 and cluster uuid3.
+        // If set to false, only IndexMetadata would be different
+        // Adding difference in EXACTLY on of these randomly will help us test if our uuid trimming logic compares both
+        // IndexMetadata and Metadata when deciding if the remote state b/w two different cluster uuids is same.
+        List uploadedIndexMetadataList3 = differGlobalMetadata
+            ? new ArrayList<>(uploadedIndexMetadataList1)
+            : List.of(new UploadedIndexMetadata("index1", "index-uuid1", "key1"));
+        IndexMetadata indexMetadata5 = IndexMetadata.builder("index1")
+            .settings(indexSettings)
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .build();
+        Map indexMetadataMap3 = differGlobalMetadata
+            ? new HashMap<>(indexMetadataMap1)
+            : Map.of("index-uuid1", indexMetadata5);
+        Metadata metadata3 = Metadata.builder()
+            .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), !differGlobalMetadata).build())
+            .build();
 
-        List uploadedIndexMetadataList3 = List.of(new UploadedIndexMetadata("index1", "index-uuid1", "key1"));
         final ClusterMetadataManifest clusterManifest3 = generateClusterMetadataManifest(
             "cluster-uuid3",
             clusterUUIDsPointers.get("cluster-uuid3"),
             randomAlphaOfLength(10),
-            uploadedIndexMetadataList3
+            uploadedIndexMetadataList3,
+            "test-metadata3"
         );
-        IndexMetadata indexMetadata5 = IndexMetadata.builder("index1")
-            .settings(indexSettings)
-            .numberOfShards(1)
-            .numberOfReplicas(1)
-            .build();
-        Map indexMetadataMap3 = Map.of("index-uuid1", indexMetadata5);
-        mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3);
-
-        when(blobStore.blobContainer(ArgumentMatchers.any())).thenReturn(
-            uuidBlobContainer,
-            blobContainer1,
-            blobContainer1,
-            blobContainer3,
-            blobContainer3,
-            blobContainer2,
-            blobContainer2,
-            blobContainer1,
-            blobContainer2,
-            blobContainer1,
-            blobContainer2
+        mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3);
+        mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V1);
+
+        ArrayList mockBlobContainerOrderedList = new ArrayList<>(
+            List.of(blobContainer1, blobContainer1, blobContainer3, blobContainer3, blobContainer2, blobContainer2)
         );
+
+        if (differGlobalMetadata) {
+            mockBlobContainerOrderedList.addAll(
+                List.of(blobContainer3, blobContainer1, blobContainer3, blobContainer1, blobContainer1, blobContainer3)
+            );
+        }
+        mockBlobContainerOrderedList.addAll(
+            List.of(blobContainer2, blobContainer1, blobContainer2, blobContainer1, blobContainer1, blobContainer2)
+        );
+        BlobContainer[] mockBlobContainerOrderedArray = new BlobContainer[mockBlobContainerOrderedList.size()];
+        mockBlobContainerOrderedList.toArray(mockBlobContainerOrderedArray);
+        when(blobStore.blobContainer(ArgumentMatchers.any())).thenReturn(uuidBlobContainer, mockBlobContainerOrderedArray);
         when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor());
     }
 
@@ -1135,7 +1182,8 @@ private ClusterMetadataManifest generateClusterMetadataManifest(
         String clusterUUID,
         String previousClusterUUID,
         String stateUUID,
-        List uploadedIndexMetadata
+        List uploadedIndexMetadata,
+        String globalMetadataFileName
     ) {
         return ClusterMetadataManifest.builder()
             .indices(uploadedIndexMetadata)
@@ -1148,7 +1196,8 @@ private ClusterMetadataManifest generateClusterMetadataManifest(
             .previousClusterUUID(previousClusterUUID)
             .committed(true)
             .clusterUUIDCommitted(true)
-            .globalMetadataFileName("test-global-metadata")
+            .globalMetadataFileName(globalMetadataFileName)
+            .codecVersion(ClusterMetadataManifest.CODEC_V1)
             .build();
     }
 
@@ -1181,17 +1230,29 @@ private void mockBlobContainer(
         ClusterMetadataManifest clusterMetadataManifest,
         Map indexMetadataMap
     ) throws IOException {
-        BlobMetadata blobMetadata = new PlainBlobMetadata("manifestFileName", 1);
+        mockBlobContainer(blobContainer, clusterMetadataManifest, indexMetadataMap, ClusterMetadataManifest.CODEC_V0);
+    }
+
+    private void mockBlobContainer(
+        BlobContainer blobContainer,
+        ClusterMetadataManifest clusterMetadataManifest,
+        Map indexMetadataMap,
+        int codecVersion
+    ) throws IOException {
+        String manifestFileName = codecVersion >= ClusterMetadataManifest.CODEC_V1
+            ? "manifest__manifestFileName__abcd__abcd__abcd__1"
+            : "manifestFileName";
+        BlobMetadata blobMetadata = new PlainBlobMetadata(manifestFileName, 1);
         when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC))
             .thenReturn(Arrays.asList(blobMetadata));
 
         BytesReference bytes = RemoteClusterStateService.CLUSTER_METADATA_MANIFEST_FORMAT.serialize(
             clusterMetadataManifest,
-            "manifestFileName",
+            manifestFileName,
             blobStoreRepository.getCompressor(),
             FORMAT_PARAMS
         );
-        when(blobContainer.readBlob("manifestFileName")).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
+        when(blobContainer.readBlob(manifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
 
         clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> {
             try {
@@ -1200,15 +1261,15 @@ private void mockBlobContainer(
                     return;
                 }
                 String fileName = uploadedIndexMetadata.getUploadedFilename();
-                BytesReference bytesIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.serialize(
-                    indexMetadata,
-                    fileName,
-                    blobStoreRepository.getCompressor(),
-                    FORMAT_PARAMS
-                );
-                when(blobContainer.readBlob(fileName + ".dat")).thenReturn(
-                    new ByteArrayInputStream(bytesIndexMetadata.streamInput().readAllBytes())
-                );
+                when(blobContainer.readBlob(fileName + ".dat")).thenAnswer((invocationOnMock) -> {
+                    BytesReference bytesIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.serialize(
+                        indexMetadata,
+                        fileName,
+                        blobStoreRepository.getCompressor(),
+                        FORMAT_PARAMS
+                    );
+                    return new ByteArrayInputStream(bytesIndexMetadata.streamInput().readAllBytes());
+                });
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
@@ -1238,15 +1299,17 @@ private void mockBlobContainerForGlobalMetadata(
         );
         when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
 
-        BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize(
-            metadata,
-            "global-metadata-file",
-            blobStoreRepository.getCompressor(),
-            FORMAT_PARAMS
-        );
         String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/");
-        when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))).thenReturn(
-            new ByteArrayInputStream(bytesGlobalMetadata.streamInput().readAllBytes())
+        when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))).thenAnswer(
+            (invocationOnMock) -> {
+                BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize(
+                    metadata,
+                    "global-metadata-file",
+                    blobStoreRepository.getCompressor(),
+                    FORMAT_PARAMS
+                );
+                return new ByteArrayInputStream(bytesGlobalMetadata.streamInput().readAllBytes());
+            }
         );
     }
 

From e618b9bd27af47c4b1a04f4eaabc323a39573c40 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Sun, 22 Oct 2023 13:28:50 +0530
Subject: [PATCH 212/388] Fix custom metadata not getting stored for remote
 store not supporting async write (#10812)

Signed-off-by: Dhwanil Patel 
---
 .../RemoteStoreClusterStateRestoreIT.java     | 28 +++++++++++--------
 .../remote/RemoteClusterStateService.java     |  8 +++++-
 .../blobstore/ChecksumBlobStoreFormat.java    | 25 +++++++++++++++--
 3 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index c2cb7cc60f152..f0863966fa222 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -25,6 +25,7 @@
 
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -233,8 +234,7 @@ public void testFullClusterRestoreGlobalMetadata() throws Exception {
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
 
         // Create global metadata - register a custom repo
-        // TODO - uncomment after all customs is also uploaded for all repos - https://github.com/opensearch-project/OpenSearch/issues/10691
-        // registerCustomRepository();
+        Path repoPath = registerCustomRepository();
 
         // Create global metadata - persistent settings
         updatePersistentSettings(Settings.builder().put(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey(), 34).build());
@@ -263,30 +263,36 @@ public void testFullClusterRestoreGlobalMetadata() throws Exception {
         verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, false);
 
         // validate global metadata restored
-        verifyRestoredRepositories();
+        verifyRestoredRepositories(repoPath);
         verifyRestoredIndexTemplate();
     }
 
-    private void registerCustomRepository() {
+    private Path registerCustomRepository() {
+        Path path = randomRepoPath();
         assertAcked(
             client().admin()
                 .cluster()
                 .preparePutRepository("custom-repo")
                 .setType("fs")
-                .setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", false))
+                .setSettings(Settings.builder().put("location", path).put("compress", false))
                 .get()
         );
+        return path;
     }
 
-    private void verifyRestoredRepositories() {
+    private void verifyRestoredRepositories(Path repoPath) {
         RepositoriesMetadata repositoriesMetadata = clusterService().state().metadata().custom(RepositoriesMetadata.TYPE);
-        assertEquals(2, repositoriesMetadata.repositories().size()); // includes remote store repo as well
+        assertEquals(3, repositoriesMetadata.repositories().size()); // includes remote store repo as well
         assertTrue(SYSTEM_REPOSITORY_SETTING.get(repositoriesMetadata.repository(REPOSITORY_NAME).settings()));
         assertTrue(SYSTEM_REPOSITORY_SETTING.get(repositoriesMetadata.repository(REPOSITORY_2_NAME).settings()));
-        // TODO - uncomment after all customs is also uploaded for all repos - https://github.com/opensearch-project/OpenSearch/issues/10691
-        // assertEquals("fs", repositoriesMetadata.repository("custom-repo").type());
-        // assertEquals(Settings.builder().put("location", randomRepoPath()).put("compress", false).build(),
-        // repositoriesMetadata.repository("custom-repo").settings());
+        assertEquals("fs", repositoriesMetadata.repository("custom-repo").type());
+        assertEquals(
+            Settings.builder().put("location", repoPath).put("compress", false).build(),
+            repositoriesMetadata.repository("custom-repo").settings()
+        );
+
+        // repo cleanup post verification
+        clusterAdmin().prepareDeleteRepository("custom-repo").get();
     }
 
     private void addClusterLevelReadOnlyBlock() throws InterruptedException, ExecutionException {
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 13fbda77d8a2d..dfe3659c1efd1 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -568,7 +568,13 @@ private ClusterMetadataManifest uploadManifest(
     private void writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest, String fileName)
         throws IOException {
         final BlobContainer metadataManifestContainer = manifestContainer(clusterName, clusterUUID);
-        CLUSTER_METADATA_MANIFEST_FORMAT.write(uploadManifest, metadataManifestContainer, fileName, blobStoreRepository.getCompressor());
+        CLUSTER_METADATA_MANIFEST_FORMAT.write(
+            uploadManifest,
+            metadataManifestContainer,
+            fileName,
+            blobStoreRepository.getCompressor(),
+            FORMAT_PARAMS
+        );
     }
 
     private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) {
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 17cb68f798094..e280141c12bc1 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -170,8 +170,29 @@ public T deserialize(String blobName, NamedXContentRegistry namedXContentRegistr
      * @param compressor          whether to use compression
      */
     public void write(final T obj, final BlobContainer blobContainer, final String name, final Compressor compressor) throws IOException {
+        write(obj, blobContainer, name, compressor, SNAPSHOT_ONLY_FORMAT_PARAMS);
+    }
+
+    /**
+     * Writes blob with resolving the blob name using {@link #blobName} method.
+     * 
                              
+     * The blob will optionally by compressed.
+     *
+     * @param obj                 object to be serialized
+     * @param blobContainer       blob container
+     * @param name                blob name
+     * @param compressor          whether to use compression
+     * @param params              ToXContent params
+     */
+    public void write(
+        final T obj,
+        final BlobContainer blobContainer,
+        final String name,
+        final Compressor compressor,
+        final ToXContent.Params params
+    ) throws IOException {
         final String blobName = blobName(name);
-        final BytesReference bytes = serialize(obj, blobName, compressor, SNAPSHOT_ONLY_FORMAT_PARAMS);
+        final BytesReference bytes = serialize(obj, blobName, compressor, params);
         blobContainer.writeBlob(blobName, bytes.streamInput(), bytes.length(), false);
     }
 
@@ -195,7 +216,7 @@ public void writeAsync(
         final ToXContent.Params params
     ) throws IOException {
         if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) {
-            write(obj, blobContainer, name, compressor);
+            write(obj, blobContainer, name, compressor, params);
             listener.onResponse(null);
             return;
         }

From a09047a4a9870eca642577094c9793256cbe71d1 Mon Sep 17 00:00:00 2001
From: Shivansh Arora <31575408+shiv0408@users.noreply.github.com>
Date: Sun, 22 Oct 2023 13:41:50 +0530
Subject: [PATCH 213/388] Removed unnecessary catch statement (#10783)

* Removed unnecessary catch statement related to repo missing exception in remote state flow

Signed-off-by: Shivansh Arora 
---
 .../main/java/org/opensearch/gateway/GatewayMetaState.java | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
index 9eb7fb0ca04d0..5d417ce78fe5c 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -67,7 +67,6 @@
 import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult;
 import org.opensearch.node.Node;
 import org.opensearch.plugins.MetadataUpgrader;
-import org.opensearch.repositories.RepositoryMissingException;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
@@ -712,12 +711,6 @@ assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) ==
                 assert verifyManifestAndClusterState(manifest, clusterState) == true : "Manifest and ClusterState are not in sync";
                 lastAcceptedManifest = manifest;
                 lastAcceptedState = clusterState;
-            } catch (RepositoryMissingException e) {
-                // TODO This logic needs to be modified once PR for repo registration during bootstrap is pushed
-                // https://github.com/opensearch-project/OpenSearch/pull/9105/
-                // After the above PR is pushed, we can remove this silent failure and throw the exception instead.
-                logger.error("Remote repository is not yet registered");
-                lastAcceptedState = clusterState;
             } catch (Exception e) {
                 handleExceptionOnWrite(e);
             }

From 9b7a9d0026aa379537804fd24b95619abe88e1c0 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Sun, 22 Oct 2023 16:16:10 +0530
Subject: [PATCH 214/388] Override local disk state if we are able to restore
 from remote (#10748)

* Override local disk state if we are able to restore from remote

Signed-off-by: bansvaru 
---
 .../RemoteStoreClusterStateRestoreIT.java     | 68 +++++++++++++
 .../opensearch/gateway/GatewayMetaState.java  |  7 +-
 .../recovery/RemoteStoreRestoreService.java   | 95 +++----------------
 3 files changed, 89 insertions(+), 81 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index f0863966fa222..29786158bc73c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -21,6 +21,7 @@
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
 import org.opensearch.gateway.remote.RemoteClusterStateService;
+import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.io.IOException;
@@ -32,6 +33,7 @@
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
 
+import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING;
 import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING;
 import static org.opensearch.cluster.metadata.Metadata.CLUSTER_READ_ONLY_BLOCK;
 import static org.opensearch.cluster.metadata.Metadata.SETTING_READ_ONLY_SETTING;
@@ -94,6 +96,72 @@ public void testFullClusterRestore() throws Exception {
         verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true);
     }
 
+    /**
+     * This test scenario covers the case where right after remote state restore and persisting it to disk via LucenePersistedState, full cluster restarts.
+     * This is a special case for remote state as at this point cluster uuid in the restored state is still ClusterState.UNKNOWN_UUID as we persist it disk.
+     * After restart the local disk state will be read but should be again overridden with remote state.
+     *
+     * 1. Form a cluster and index few docs
+     * 2. Replace all nodes to remove all local disk state
+     * 3. Start cluster manager node without correct seeding to ensure local disk state is written with cluster uuid ClusterState.UNKNOWN_UUID but with remote restored Metadata
+     * 4. Restart the cluster manager node with correct seeding.
+     * 5. After restart the cluster manager picks up the local disk state with has same Metadata as remote but cluster uuid is still ClusterState.UNKNOWN_UUID
+     * 6. The cluster manager will try to restore from remote again.
+     * 7. Metadata loaded from local disk state will be overridden with remote Metadata and no conflict should arise.
+     * 8. Add data nodes to recover index data
+     * 9. Verify Metadata and index data is restored.
+     */
+    public void testFullClusterStateRestore() throws Exception {
+        int shardCount = randomIntBetween(1, 2);
+        int replicaCount = 1;
+        int dataNodeCount = shardCount * (replicaCount + 1);
+        int clusterManagerNodeCount = 1;
+
+        // index some data to generate files in remote directory
+        Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, 1);
+        String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+
+        // stop all nodes
+        internalCluster().stopAllNodes();
+
+        // start a cluster manager node with no cluster manager seeding.
+        // This should fail with IllegalStateException as cluster manager fails to form without any initial seed
+        assertThrows(
+            IllegalStateException.class,
+            () -> internalCluster().startClusterManagerOnlyNodes(
+                clusterManagerNodeCount,
+                Settings.builder()
+                    .putList(INITIAL_CLUSTER_MANAGER_NODES_SETTING.getKey()) // disable seeding during bootstrapping
+                    .build()
+            )
+        );
+
+        // verify cluster manager not elected
+        String newClusterUUID = clusterService().state().metadata().clusterUUID();
+        assert Objects.equals(newClusterUUID, ClusterState.UNKNOWN_UUID)
+            : "Disabling Cluster manager seeding failed. cluster uuid is not unknown";
+
+        // restart cluster manager with correct seed
+        internalCluster().fullRestart(new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) {
+                return Settings.builder()
+                    .putList(INITIAL_CLUSTER_MANAGER_NODES_SETTING.getKey(), nodeName)  // Seed with correct Cluster Manager node
+                    .build();
+            }
+        });
+
+        // validate new cluster state formed
+        newClusterUUID = clusterService().state().metadata().clusterUUID();
+        assert !Objects.equals(newClusterUUID, ClusterState.UNKNOWN_UUID) : "cluster restart not successful. cluster uuid is still unknown";
+        assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
+        validateMetadata(List.of(INDEX_NAME));
+
+        // start data nodes to trigger index data recovery
+        internalCluster().startDataOnlyNodes(dataNodeCount);
+        verifyRestoredData(indexStats, INDEX_NAME);
+    }
+
     public void testFullClusterRestoreMultipleIndices() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
index 5d417ce78fe5c..f855449c708d2 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -174,7 +174,9 @@ public void start(
                             if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) {
                                 // Load state from remote
                                 final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore(
-                                    clusterState,
+                                    // Remote Metadata should always override local disk Metadata
+                                    // if local disk Metadata's cluster uuid is UNKNOWN_UUID
+                                    ClusterState.builder(clusterState).metadata(Metadata.EMPTY_METADATA).build(),
                                     lastKnownClusterUUID,
                                     false,
                                     new String[] {}
@@ -549,6 +551,9 @@ static class LucenePersistedState implements PersistedState {
             // out by this version of OpenSearch. TODO TBD should we avoid indexing when possible?
             final PersistedClusterStateService.Writer writer = persistedClusterStateService.createWriter();
             try {
+                // During remote state restore, there will be non empty metadata getting persisted with cluster UUID as
+                // ClusterState.UNKOWN_UUID . The valid UUID will be generated and persisted along with the first cluster state getting
+                // published.
                 writer.writeFullStateAndCommit(currentTerm, lastAcceptedState);
             } catch (Exception e) {
                 try {
diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
index 6692d521b8f65..9541d13421e27 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
@@ -40,12 +40,10 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -146,6 +144,11 @@ public RemoteRestoreResult restore(
             || restoreClusterUUID.isBlank()) == false;
         if (metadataFromRemoteStore) {
             try {
+                // Restore with current cluster UUID will fail as same indices would be present in the cluster which we are trying to
+                // restore
+                if (currentState.metadata().clusterUUID().equals(restoreClusterUUID)) {
+                    throw new IllegalArgumentException("clusterUUID to restore from should be different from current cluster UUID");
+                }
                 remoteMetadata = remoteClusterStateService.getLatestMetadata(currentState.getClusterName().value(), restoreClusterUUID);
                 remoteMetadata.getIndices().values().forEach(indexMetadata -> {
                     indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));
@@ -158,12 +161,21 @@ public RemoteRestoreResult restore(
                 IndexMetadata indexMetadata = currentState.metadata().index(indexName);
                 if (indexMetadata == null) {
                     logger.warn("Index restore is not supported for non-existent index. Skipping: {}", indexName);
+                } else if (indexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false) == false) {
+                    logger.warn("Remote store is not enabled for index: {}", indexName);
+                } else if (restoreAllShards && IndexMetadata.State.CLOSE.equals(indexMetadata.getState()) == false) {
+                    throw new IllegalStateException(
+                        String.format(
+                            Locale.ROOT,
+                            "cannot restore index [%s] because an open index with same name/uuid already exists in the cluster.",
+                            indexName
+                        ) + " Close the existing index."
+                    );
                 } else {
                     indexMetadataMap.put(indexName, new Tuple<>(false, indexMetadata));
                 }
             }
         }
-        validate(currentState, indexMetadataMap, restoreClusterUUID, restoreAllShards);
         return executeRestore(currentState, indexMetadataMap, restoreAllShards, remoteMetadata);
     }
 
@@ -272,83 +284,6 @@ private void restoreGlobalMetadata(Metadata.Builder mdBuilder, Metadata remoteMe
         repositoriesMetadata.ifPresent(metadata -> mdBuilder.putCustom(RepositoriesMetadata.TYPE, metadata));
     }
 
-    /**
-     * Performs various validations needed before executing restore
-     * @param currentState current cluster state
-     * @param indexMetadataMap map of index metadata to restore
-     * @param restoreClusterUUID cluster UUID used to restore IndexMetadata
-     * @param restoreAllShards indicates if all shards of the index needs to be restored. This flat is ignored if remoteClusterUUID is provided
-     */
-    private void validate(
-        ClusterState currentState,
-        Map> indexMetadataMap,
-        @Nullable String restoreClusterUUID,
-        boolean restoreAllShards
-    ) throws IllegalStateException, IllegalArgumentException {
-        String errorMsg = "cannot restore index [%s] because an open index with same name/uuid already exists in the cluster.";
-
-        // Restore with current cluster UUID will fail as same indices would be present in the cluster which we are trying to
-        // restore
-        if (currentState.metadata().clusterUUID().equals(restoreClusterUUID)) {
-            throw new IllegalArgumentException("clusterUUID to restore from should be different from current cluster UUID");
-        }
-        for (Map.Entry> indexMetadataEntry : indexMetadataMap.entrySet()) {
-            String indexName = indexMetadataEntry.getKey();
-            IndexMetadata indexMetadata = indexMetadataEntry.getValue().v2();
-            String indexUUID = indexMetadata.getIndexUUID();
-            boolean metadataFromRemoteStore = indexMetadataEntry.getValue().v1();
-            if (indexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false)) {
-                if (metadataFromRemoteStore) {
-                    Set graveyardIndexNames = new HashSet<>();
-                    Set graveyardIndexUUID = new HashSet<>();
-                    Set liveClusterIndexUUIDs = currentState.metadata()
-                        .indices()
-                        .values()
-                        .stream()
-                        .map(IndexMetadata::getIndexUUID)
-                        .collect(Collectors.toSet());
-
-                    currentState.metadata().indexGraveyard().getTombstones().forEach(tombstone -> {
-                        graveyardIndexNames.add(tombstone.getIndex().getName());
-                        graveyardIndexUUID.add(tombstone.getIndex().getUUID());
-                    });
-
-                    // Since updates to graveyard are synced to remote we should neven land in a situation where remote contain index
-                    // metadata for graveyard index.
-                    assert graveyardIndexNames.contains(indexName) == false : String.format(
-                        Locale.ROOT,
-                        "Index name [%s] exists in graveyard!",
-                        indexName
-                    );
-                    assert graveyardIndexUUID.contains(indexUUID) == false : String.format(
-                        Locale.ROOT,
-                        "Index UUID [%s] exists in graveyard!",
-                        indexUUID
-                    );
-
-                    // Any indices being restored from remote cluster state should not already be part of the cluster as this causes
-                    // conflict
-                    boolean sameNameIndexExists = currentState.metadata().hasIndex(indexName);
-                    boolean sameUUIDIndexExists = liveClusterIndexUUIDs.contains(indexUUID);
-                    if (sameNameIndexExists || sameUUIDIndexExists) {
-                        String finalErrorMsg = String.format(Locale.ROOT, errorMsg, indexName);
-                        logger.info(finalErrorMsg);
-                        throw new IllegalStateException(finalErrorMsg);
-                    }
-
-                    boolean isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(indexMetadata.getSettings());
-                    createIndexService.validateIndexName(indexName, currentState);
-                    createIndexService.validateDotIndex(indexName, isHidden);
-                    shardLimitValidator.validateShardLimit(indexName, indexMetadata.getSettings(), currentState);
-                } else if (restoreAllShards && IndexMetadata.State.CLOSE.equals(indexMetadata.getState()) == false) {
-                    throw new IllegalStateException(String.format(Locale.ROOT, errorMsg, indexName) + " Close the existing index.");
-                }
-            } else {
-                logger.warn("Remote store is not enabled for index: {}", indexName);
-            }
-        }
-    }
-
     /**
      * Result of a remote restore operation.
      */

From 5e7c0689534f94b5f09f85eb0a64b42f79899241 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Mon, 23 Oct 2023 11:24:13 +0530
Subject: [PATCH 215/388] fix integ test: check for red indices and then
 manually restore to green (#10838)

Signed-off-by: bansvaru 
---
 .../remotestore/RemoteStoreClusterStateRestoreIT.java         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index 29786158bc73c..c429b587a1399 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -111,7 +111,7 @@ public void testFullClusterRestore() throws Exception {
      * 8. Add data nodes to recover index data
      * 9. Verify Metadata and index data is restored.
      */
-    public void testFullClusterStateRestore() throws Exception {
+    public void testFullClusterRestoreDoesntFailWithConflictingLocalState() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
         int dataNodeCount = shardCount * (replicaCount + 1);
@@ -159,7 +159,7 @@ public Settings onNodeStopped(String nodeName) {
 
         // start data nodes to trigger index data recovery
         internalCluster().startDataOnlyNodes(dataNodeCount);
-        verifyRestoredData(indexStats, INDEX_NAME);
+        verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true);
     }
 
     public void testFullClusterRestoreMultipleIndices() throws Exception {

From ef4b327ab48a6a674b657049e440418fe427cb0c Mon Sep 17 00:00:00 2001
From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com>
Date: Sun, 22 Oct 2023 22:54:43 -0700
Subject: [PATCH 216/388] Update version check made in PR #10656. (#10835)

Signed-off-by: Rishikesh1159 
---
 .../action/admin/cluster/node/stats/NodeStats.java         | 7 +++----
 .../opensearch/index/SegmentReplicationRejectionStats.java | 6 ++----
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index 3d37056956c69..874713b51d627 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -215,8 +215,7 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             resourceUsageStats = null;
         }
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             segmentReplicationRejectionStats = in.readOptionalWriteable(SegmentReplicationRejectionStats::new);
         } else {
             segmentReplicationRejectionStats = null;
@@ -431,6 +430,7 @@ public SegmentReplicationRejectionStats getSegmentReplicationRejectionStats() {
         return segmentReplicationRejectionStats;
     }
 
+    @Nullable
     public RepositoriesStats getRepositoriesStats() {
         return repositoriesStats;
     }
@@ -481,8 +481,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(resourceUsageStats);
         }
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(segmentReplicationRejectionStats);
         }
         if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java
index 9f9f150ebe2d7..492f253bbcb7c 100644
--- a/server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationRejectionStats.java
@@ -34,8 +34,7 @@ public SegmentReplicationRejectionStats(final long totalRejectionCount) {
     }
 
     public SegmentReplicationRejectionStats(StreamInput in) throws IOException {
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             this.totalRejectionCount = in.readVLong();
         }
     }
@@ -53,8 +52,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeVLong(totalRejectionCount);
         }
     }

From b5ef0788699152fb2b1ae158b518c212ba784184 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Mon, 23 Oct 2023 13:56:33 +0530
Subject: [PATCH 217/388] Fix remote cluster restore for data stream (#10777)

* Fix remote cluster restore for data stream, also added integ test

Signed-off-by: Dhwanil Patel 
---
 .../datastream/DataStreamTestCase.java        |  3 ++-
 .../RemoteStoreClusterStateRestoreIT.java     |  9 +++++++
 .../remote/RemoteClusterStateService.java     |  5 +++-
 .../RemoteClusterStateServiceTests.java       |  6 ++---
 .../opensearch/test/InternalTestCluster.java  | 12 ++++++++++
 .../test/OpenSearchIntegTestCase.java         | 24 +++++++++++++++++++
 6 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java
index 50ff76c6b62f3..82ab5b0118c0e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java
@@ -37,6 +37,7 @@ public AcknowledgedResponse createDataStream(String name) throws Exception {
         CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(name);
         AcknowledgedResponse response = client().admin().indices().createDataStream(request).get();
         assertThat(response.isAcknowledged(), is(true));
+        performRemoteStoreTestAction();
         return response;
     }
 
@@ -67,6 +68,7 @@ public RolloverResponse rolloverDataStream(String name) throws Exception {
         RolloverResponse response = client().admin().indices().rolloverIndex(request).get();
         assertThat(response.isAcknowledged(), is(true));
         assertThat(response.isRolledOver(), is(true));
+        performRemoteStoreTestAction();
         return response;
     }
 
@@ -109,5 +111,4 @@ public AcknowledgedResponse deleteIndexTemplate(String name) throws Exception {
         assertThat(response.isAcknowledged(), is(true));
         return response;
     }
-
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index c429b587a1399..e9afd6d36bb87 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -9,6 +9,7 @@
 package org.opensearch.remotestore;
 
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.datastream.DataStreamRolloverIT;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.opensearch.cluster.ClusterState;
@@ -291,6 +292,14 @@ private void validateCurrentMetadata() throws Exception {
         });
     }
 
+    public void testDataStreamPostRemoteStateRestore() throws Exception {
+        new DataStreamRolloverIT() {
+            protected boolean triggerRemoteStateRestore() {
+                return true;
+            }
+        }.testDataStreamRollover();
+    }
+
     public void testFullClusterRestoreGlobalMetadata() throws Exception {
         int shardCount = randomIntBetween(1, 2);
         int replicaCount = 1;
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index dfe3659c1efd1..b505b3e2d4c0c 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -732,7 +732,10 @@ public Metadata getLatestMetadata(String clusterName, String clusterUUID) {
         // Fetch Index Metadata
         Map indices = getIndexMetadataMap(clusterName, clusterUUID, clusterMetadataManifest.get());
 
-        return Metadata.builder(globalMetadata).indices(indices).build();
+        Map indexMetadataMap = new HashMap<>();
+        indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); });
+
+        return Metadata.builder(globalMetadata).indices(indexMetadataMap).build();
     }
 
     private Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) {
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 55094400581b4..827735f6bcc0b 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -826,9 +826,9 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
         ).getIndices();
 
         assertEquals(indexMetadataMap.size(), 1);
-        assertEquals(indexMetadataMap.get(index.getUUID()).getIndex().getName(), index.getName());
-        assertEquals(indexMetadataMap.get(index.getUUID()).getNumberOfShards(), indexMetadata.getNumberOfShards());
-        assertEquals(indexMetadataMap.get(index.getUUID()).getNumberOfReplicas(), indexMetadata.getNumberOfReplicas());
+        assertEquals(indexMetadataMap.get(index.getName()).getIndex().getName(), index.getName());
+        assertEquals(indexMetadataMap.get(index.getName()).getNumberOfShards(), indexMetadata.getNumberOfShards());
+        assertEquals(indexMetadataMap.get(index.getName()).getNumberOfReplicas(), indexMetadata.getNumberOfReplicas());
     }
 
     public void testMarkLastStateAsCommittedSuccess() throws IOException {
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index 63d8f069bebea..952cd6c085966 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -1871,6 +1871,18 @@ public void stopAllNodes() {
         }
     }
 
+    /**
+     * Replace all nodes by stopping all current node and starting new node.
+     * Used for remote store test cases, where remote state is restored.
+     */
+    public void resetCluster() {
+        int totalClusterManagerNodes = numClusterManagerNodes();
+        int totalDataNodes = numDataNodes();
+        stopAllNodes();
+        startClusterManagerOnlyNodes(totalClusterManagerNodes);
+        startDataOnlyNodes(totalDataNodes);
+    }
+
     private synchronized void startAndPublishNodesAndClients(List nodeAndClients) {
         if (nodeAndClients.size() > 0) {
             final int newClusterManagers = (int) nodeAndClients.stream()
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index c16cc1d2a5fba..ad27d9834f159 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -789,6 +789,30 @@ protected Settings featureFlagSettings() {
         return featureSettings.build();
     }
 
+    /**
+     * Represent if it needs to trigger remote state restore or not.
+     * For tests with remote store enabled domain, it will be overridden to true.
+     *
+     * @return if needs to perform remote state restore or not
+     */
+    protected boolean triggerRemoteStateRestore() {
+        return false;
+    }
+
+    /**
+     * For tests with remote cluster state, it will reset the cluster and cluster state will be
+     * restored from remote.
+     */
+    protected void performRemoteStoreTestAction() {
+        if (triggerRemoteStateRestore()) {
+            String clusterUUIDBefore = clusterService().state().metadata().clusterUUID();
+            internalCluster().resetCluster();
+            String clusterUUIDAfter = clusterService().state().metadata().clusterUUID();
+            // assertion that UUID is changed post restore.
+            assertFalse(clusterUUIDBefore.equals(clusterUUIDAfter));
+        }
+    }
+
     /**
      * Creates one or more indices and asserts that the indices are acknowledged. If one of the indices
      * already exists this method will fail and wipe all the indices created so far.

From 7453daa2b7ce698fa7bf3647002eb12394a5dae1 Mon Sep 17 00:00:00 2001
From: Ashish 
Date: Mon, 23 Oct 2023 14:10:14 +0530
Subject: [PATCH 218/388] [Remote Store] Sync segments in refresh listener on
 refresh after commit (#10830)

* [Remote Store] Sync segments in refresh listener on refresh after commit

Signed-off-by: Ashish Singh 

* Add Integration Tests

Signed-off-by: Ashish Singh 

* Add comments and java doc

Signed-off-by: Ashish Singh 

---------

Signed-off-by: Ashish Singh 
---
 .../remotestore/RemoteStoreStatsIT.java       | 76 ++++++++++++++++-
 .../remote/RemoteTranslogTransferTracker.java | 57 +++++++++++++
 .../opensearch/index/shard/IndexShard.java    |  8 ++
 .../shard/RemoteStoreRefreshListener.java     | 84 +++++++++++++++----
 .../index/translog/RemoteFsTranslog.java      |  8 +-
 5 files changed, 215 insertions(+), 18 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
index 8ae25c6758195..5e91176ed0473 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
@@ -15,6 +15,8 @@
 import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.coordination.FollowersChecker;
+import org.opensearch.cluster.coordination.LeaderChecker;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.ShardRoutingState;
@@ -23,15 +25,20 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.remote.RemoteSegmentTransferTracker;
 import org.opensearch.index.remote.RemoteTranslogTransferTracker;
+import org.opensearch.plugins.Plugin;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
-import org.junit.Before;
+import org.opensearch.test.disruption.NetworkDisruption;
+import org.opensearch.test.transport.MockTransportService;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -44,12 +51,17 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase {
 
     private static final String INDEX_NAME = "remote-store-test-idx-1";
 
-    @Before
+    @Override
+    protected Collection> nodePlugins() {
+        return Arrays.asList(MockTransportService.TestPlugin.class);
+    }
+
     public void setup() {
         internalCluster().startNodes(3);
     }
 
     public void testStatsResponseFromAllNodes() {
+        setup();
 
         // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes
         // during this time frame. This ensures that the segment upload has started.
@@ -118,6 +130,7 @@ public void testStatsResponseFromAllNodes() {
     }
 
     public void testStatsResponseAllShards() {
+        setup();
 
         // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes
         // during this time frame. This ensures that the segment upload has started.
@@ -175,6 +188,7 @@ public void testStatsResponseAllShards() {
     }
 
     public void testStatsResponseFromLocalNode() {
+        setup();
 
         // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes
         // during this time frame. This ensures that the segment upload has started.
@@ -236,6 +250,7 @@ public void testStatsResponseFromLocalNode() {
     }
 
     public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exception {
+        setup();
         // Scenario:
         // - Create index with single primary and single replica shard
         // - Disable Refresh Interval for the index
@@ -325,6 +340,7 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce
     }
 
     public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() throws Exception {
+        setup();
         // Scenario:
         // - Create index with single primary and N-1 replica shards (N = no of data nodes)
         // - Disable Refresh Interval for the index
@@ -416,6 +432,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr
     }
 
     public void testStatsOnShardRelocation() {
+        setup();
         // Scenario:
         // - Create index with single primary and single replica shard
         // - Index documents
@@ -471,6 +488,7 @@ public void testStatsOnShardRelocation() {
     }
 
     public void testStatsOnShardUnassigned() throws IOException {
+        setup();
         // Scenario:
         // - Create index with single primary and two replica shard
         // - Index documents
@@ -497,6 +515,7 @@ public void testStatsOnShardUnassigned() throws IOException {
     }
 
     public void testStatsOnRemoteStoreRestore() throws IOException {
+        setup();
         // Creating an index with primary shard count == total nodes in cluster and 0 replicas
         int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes();
         createIndex(INDEX_NAME, remoteStoreIndexSettings(0, dataNodeCount));
@@ -544,6 +563,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException {
     }
 
     public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exception {
+        setup();
         // Create an index with one primary and one replica shard
         createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1));
         ensureGreen(INDEX_NAME);
@@ -581,6 +601,58 @@ public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exce
         }, 5, TimeUnit.SECONDS);
     }
 
+    public void testStatsCorrectnessOnFailover() {
+        Settings clusterSettings = Settings.builder()
+            .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "100ms")
+            .put(LeaderChecker.LEADER_CHECK_INTERVAL_SETTING.getKey(), "500ms")
+            .put(LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), 1)
+            .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "100ms")
+            .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "500ms")
+            .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1)
+            .put(nodeSettings(0))
+            .build();
+        String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(clusterSettings);
+        internalCluster().startDataOnlyNodes(2, clusterSettings);
+
+        // Create an index with one primary and one replica shard
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1));
+        ensureGreen(INDEX_NAME);
+
+        // Index some docs and refresh
+        indexDocs();
+        refresh(INDEX_NAME);
+
+        String primaryNode = primaryNodeName(INDEX_NAME);
+        String replicaNode = replicaNodeName(INDEX_NAME);
+
+        // Start network disruption - primary node will be isolated
+        Set nodesInOneSide = Stream.of(clusterManagerNode, replicaNode).collect(Collectors.toCollection(HashSet::new));
+        Set nodesInOtherSide = Stream.of(primaryNode).collect(Collectors.toCollection(HashSet::new));
+        NetworkDisruption networkDisruption = new NetworkDisruption(
+            new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInOtherSide),
+            NetworkDisruption.DISCONNECT
+        );
+        internalCluster().setDisruptionScheme(networkDisruption);
+        logger.info("--> network disruption is started");
+        networkDisruption.startDisrupting();
+        ensureStableCluster(2, clusterManagerNode);
+
+        RemoteStoreStatsResponse response = client(clusterManagerNode).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get();
+        final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, "0");
+        List matches = Arrays.stream(response.getRemoteStoreStats())
+            .filter(stat -> indexShardId.equals(stat.getSegmentStats().shardId.toString()))
+            .collect(Collectors.toList());
+        assertEquals(1, matches.size());
+        RemoteSegmentTransferTracker.Stats segmentStats = matches.get(0).getSegmentStats();
+        assertEquals(0, segmentStats.refreshTimeLagMs);
+
+        networkDisruption.stopDisrupting();
+        internalCluster().clearDisruptionScheme();
+        ensureStableCluster(3, clusterManagerNode);
+        ensureGreen(INDEX_NAME);
+        logger.info("Test completed");
+    }
+
     private void indexDocs() {
         for (int i = 0; i < randomIntBetween(5, 10); i++) {
             if (randomBoolean()) {
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java
index 1a9896540212e..4214a87049350 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java
@@ -232,6 +232,63 @@ public RemoteTranslogTransferTracker.Stats stats() {
         );
     }
 
+    @Override
+    public String toString() {
+        return "RemoteTranslogTransferStats{"
+            + "lastSuccessfulUploadTimestamp="
+            + lastSuccessfulUploadTimestamp.get()
+            + ","
+            + "totalUploadsStarted="
+            + totalUploadsStarted.get()
+            + ","
+            + "totalUploadsSucceeded="
+            + totalUploadsSucceeded.get()
+            + ","
+            + "totalUploadsFailed="
+            + totalUploadsFailed.get()
+            + ","
+            + "uploadBytesStarted="
+            + uploadBytesStarted.get()
+            + ","
+            + "uploadBytesFailed="
+            + uploadBytesFailed.get()
+            + ","
+            + "totalUploadTimeInMillis="
+            + totalUploadTimeInMillis.get()
+            + ","
+            + "uploadBytesMovingAverage="
+            + uploadBytesMovingAverageReference.get().getAverage()
+            + ","
+            + "uploadBytesPerSecMovingAverage="
+            + uploadBytesPerSecMovingAverageReference.get().getAverage()
+            + ","
+            + "uploadTimeMovingAverage="
+            + uploadTimeMsMovingAverageReference.get().getAverage()
+            + ","
+            + "lastSuccessfulDownloadTimestamp="
+            + lastSuccessfulDownloadTimestamp.get()
+            + ","
+            + "totalDownloadsSucceeded="
+            + totalDownloadsSucceeded.get()
+            + ","
+            + "downloadBytesSucceeded="
+            + downloadBytesSucceeded.get()
+            + ","
+            + "totalDownloadTimeInMillis="
+            + totalDownloadTimeInMillis.get()
+            + ","
+            + "downloadBytesMovingAverage="
+            + downloadBytesMovingAverageReference.get().getAverage()
+            + ","
+            + "downloadBytesPerSecMovingAverage="
+            + downloadBytesPerSecMovingAverageReference.get().getAverage()
+            + ","
+            + "downloadTimeMovingAverage="
+            + downloadTimeMsMovingAverageReference.get().getAverage()
+            + ","
+            + "}";
+    }
+
     /**
      * Represents the tracker's state as seen in the stats API.
      *
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index f990a3b56e856..fb4e9056153aa 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -4774,6 +4774,8 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal) throws IOE
      * @throws IOException if exception occurs while reading segments from remote store.
      */
     public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runnable onFileSync) throws IOException {
+        boolean syncSegmentSuccess = false;
+        long startTimeMs = System.currentTimeMillis();
         assert indexSettings.isRemoteStoreEnabled();
         logger.trace("Downloading segments from remote segment store");
         RemoteSegmentStoreDirectory remoteDirectory = getRemoteDirectory();
@@ -4823,9 +4825,15 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runn
                     : "There should not be any segments file in the dir";
                 store.commitSegmentInfos(infosSnapshot, processedLocalCheckpoint, processedLocalCheckpoint);
             }
+            syncSegmentSuccess = true;
         } catch (IOException e) {
             throw new IndexShardRecoveryException(shardId, "Exception while copying segment files from remote segment store", e);
         } finally {
+            logger.trace(
+                "syncSegmentsFromRemoteSegmentStore success={} elapsedTime={}",
+                syncSegmentSuccess,
+                (System.currentTimeMillis() - startTimeMs)
+            );
             store.decRef();
             remoteStore.decRef();
         }
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index c650edc31da8d..3e97b07abfb5d 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -123,14 +123,13 @@ public void beforeRefresh() throws IOException {}
 
     @Override
     protected void runAfterRefreshExactlyOnce(boolean didRefresh) {
-        if (shouldSync(didRefresh)) {
+        // We have 2 separate methods to check if sync needs to be done or not. This is required since we use the return boolean
+        // from isReadyForUpload to schedule refresh retries as the index shard or the primary mode are not in complete
+        // ready state.
+        if (shouldSync(didRefresh) && isReadyForUpload()) {
             segmentTracker.updateLocalRefreshTimeAndSeqNo();
             try {
-                if (this.primaryTerm != indexShard.getOperationPrimaryTerm()) {
-                    logger.debug("primaryTerm update from={} to={}", primaryTerm, indexShard.getOperationPrimaryTerm());
-                    this.primaryTerm = indexShard.getOperationPrimaryTerm();
-                    this.remoteDirectory.init();
-                }
+                initializeRemoteDirectoryOnTermUpdate();
                 try (GatedCloseable segmentInfosGatedCloseable = indexShard.getSegmentInfosSnapshot()) {
                     Collection localSegmentsPostRefresh = segmentInfosGatedCloseable.get().files(true);
                     updateLocalSizeMapAndTracker(localSegmentsPostRefresh);
@@ -160,20 +159,20 @@ protected boolean performAfterRefreshWithPermit(boolean didRefresh) {
     }
 
     private boolean shouldSync(boolean didRefresh) {
-        // The third condition exists for uploading the zero state segments where the refresh has not changed the reader reference, but it
-        // is important to upload the zero state segments so that the restore does not break.
         return this.primaryTerm != indexShard.getOperationPrimaryTerm()
+            // If the readers change, didRefresh is always true.
             || didRefresh
-            || remoteDirectory.getSegmentsUploadedToRemoteStore().isEmpty();
+            // The third condition exists for uploading the zero state segments where the refresh has not changed the reader
+            // reference, but it is important to upload the zero state segments so that the restore does not break.
+            || remoteDirectory.getSegmentsUploadedToRemoteStore().isEmpty()
+            // When the shouldSync is called the first time, then 1st condition on primary term is true. But after that
+            // we update the primary term and the same condition would not evaluate to true again in syncSegments.
+            // Below check ensures that if there is commit, then that gets picked up by both 1st and 2nd shouldSync call.
+            || isRefreshAfterCommitSafe();
     }
 
     private boolean syncSegments() {
-        if (indexShard.getReplicationTracker().isPrimaryMode() == false || indexShard.state() == IndexShardState.CLOSED) {
-            logger.debug(
-                "Skipped syncing segments with primaryMode={} indexShardState={}",
-                indexShard.getReplicationTracker().isPrimaryMode(),
-                indexShard.state()
-            );
+        if (isReadyForUpload() == false) {
             // Following check is required to enable retry and make sure that we do not lose this refresh event
             // When primary shard is restored from remote store, the recovery happens first followed by changing
             // primaryMode to true. Due to this, the refresh that is triggered post replay of translog will not go through
@@ -323,6 +322,19 @@ private boolean isRefreshAfterCommit() throws IOException {
             && !remoteDirectory.containsFile(lastCommittedLocalSegmentFileName, getChecksumOfLocalFile(lastCommittedLocalSegmentFileName)));
     }
 
+    /**
+     * Returns if the current refresh has happened after a commit.
+     * @return true if this refresh has happened on account of a commit. If otherwise or exception, returns false.
+     */
+    private boolean isRefreshAfterCommitSafe() {
+        try {
+            return isRefreshAfterCommit();
+        } catch (Exception e) {
+            logger.info("Exception occurred in isRefreshAfterCommitSafe", e);
+        }
+        return false;
+    }
+
     void uploadMetadata(Collection localSegmentsPostRefresh, SegmentInfos segmentInfos, ReplicationCheckpoint replicationCheckpoint)
         throws IOException {
         final long maxSeqNo = ((InternalEngine) indexShard.getEngine()).currentOngoingRefreshCheckpoint();
@@ -439,6 +451,48 @@ private void updateFinalStatusInSegmentTracker(boolean uploadStatus, long bytesB
         }
     }
 
+    /**
+     * On primary term update, we (re)initialise the remote segment directory to reflect the latest metadata file that
+     * has been uploaded to remote store successfully. This method also updates the segment tracker about the latest
+     * uploaded segment files onto remote store.
+     */
+    private void initializeRemoteDirectoryOnTermUpdate() throws IOException {
+        if (this.primaryTerm != indexShard.getOperationPrimaryTerm()) {
+            logger.trace("primaryTerm update from={} to={}", primaryTerm, indexShard.getOperationPrimaryTerm());
+            this.primaryTerm = indexShard.getOperationPrimaryTerm();
+            RemoteSegmentMetadata uploadedMetadata = this.remoteDirectory.init();
+
+            // During failover, the uploaded metadata would have names of files that have been uploaded to remote store.
+            // Here we update the tracker with latest remote uploaded files.
+            if (uploadedMetadata != null) {
+                segmentTracker.setLatestUploadedFiles(uploadedMetadata.getMetadata().keySet());
+            }
+        }
+    }
+
+    /**
+     * This checks for readiness of the index shard and primary mode. This has separated from shouldSync since we use the
+     * returned value of this method for scheduling retries in syncSegments method.
+     * @return true iff primaryMode is true and index shard is not in closed state.
+     */
+    private boolean isReadyForUpload() {
+        boolean isReady = indexShard.getReplicationTracker().isPrimaryMode() && indexShard.state() != IndexShardState.CLOSED;
+        if (isReady == false) {
+            StringBuilder sb = new StringBuilder("Skipped syncing segments with");
+            if (indexShard.getReplicationTracker() != null) {
+                sb.append(" primaryMode=").append(indexShard.getReplicationTracker().isPrimaryMode());
+            }
+            if (indexShard.state() != null) {
+                sb.append(" indexShardState=").append(indexShard.state());
+            }
+            if (indexShard.getEngineOrNull() != null) {
+                sb.append(" engineType=").append(indexShard.getEngine().getClass().getSimpleName());
+            }
+            logger.trace(sb.toString());
+        }
+        return isReady;
+    }
+
     /**
      * Creates an {@link UploadListener} containing the stats population logic which would be triggered before and after segment upload events
      */
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
index 2dd9b1a545d4a..db85a37b556fc 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -161,6 +161,7 @@ public static void download(Repository repository, ShardId shardId, ThreadPool t
             remoteTranslogTransferTracker
         );
         RemoteFsTranslog.download(translogTransferManager, location, logger);
+        logger.trace(remoteTranslogTransferTracker.toString());
     }
 
     static void download(TranslogTransferManager translogTransferManager, Path location, Logger logger) throws IOException {
@@ -173,15 +174,20 @@ static void download(TranslogTransferManager translogTransferManager, Path locat
          */
         IOException ex = null;
         for (int i = 0; i <= DOWNLOAD_RETRIES; i++) {
+            boolean success = false;
+            long startTimeMs = System.currentTimeMillis();
             try {
                 downloadOnce(translogTransferManager, location, logger);
+                success = true;
                 return;
             } catch (FileNotFoundException | NoSuchFileException e) {
                 // continue till download retries
                 ex = e;
+            } finally {
+                logger.trace("downloadOnce success={} timeElapsed={}", success, (System.currentTimeMillis() - startTimeMs));
             }
         }
-        logger.debug("Exhausted all download retries during translog/checkpoint file download");
+        logger.info("Exhausted all download retries during translog/checkpoint file download");
         throw ex;
     }
 

From 218a2ef7c6f670f99cc20d22f8866d7597718640 Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Mon, 23 Oct 2023 14:46:26 +0530
Subject: [PATCH 219/388] [Remote Store] Add extra buffer before deleting older
 generations of translog (#10817)

---------

Signed-off-by: Gaurav Bafna 
---
 .../common/settings/IndexScopedSettings.java  |   1 +
 .../org/opensearch/index/IndexSettings.java   |  19 ++
 .../index/translog/RemoteFsTranslog.java      |   2 +-
 .../index/translog/RemoteFsTranslogTests.java | 181 +++++++++++++-----
 4 files changed, 154 insertions(+), 49 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index 83bf8c82ee3dd..62e8faf33e1fa 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -221,6 +221,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
 
                 // Settings for remote translog
                 IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
+                IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING,
 
                 // Settings for remote store enablement
                 IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING,
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 99d2b5a74c406..00e765d73f77f 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -668,6 +668,14 @@ public static IndexMergePolicy fromString(String text) {
         Property.IndexScope
     );
 
+    public static final Setting INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING = Setting.intSetting(
+        "index.remote_store.translog.keep_extra_gen",
+        100,
+        0,
+        Property.Dynamic,
+        Property.IndexScope
+    );
+
     private final Index index;
     private final Version version;
     private final Logger logger;
@@ -680,6 +688,7 @@ public static IndexMergePolicy fromString(String text) {
     private final String remoteStoreTranslogRepository;
     private final String remoteStoreRepository;
     private final boolean isRemoteSnapshot;
+    private int remoteTranslogKeepExtraGen;
     private Version extendedCompatibilitySnapshotVersion;
 
     // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock
@@ -850,6 +859,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY);
         remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
         remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY);
+        this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings);
         isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(this.settings);
 
         if (isRemoteSnapshot && FeatureFlags.isEnabled(SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY)) {
@@ -1021,6 +1031,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
             INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
             this::setRemoteTranslogUploadBufferInterval
         );
+        scopedSettings.addSettingsUpdateConsumer(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING, this::setRemoteTranslogKeepExtraGen);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1300,6 +1311,10 @@ public TimeValue getRemoteTranslogUploadBufferInterval() {
         return remoteTranslogUploadBufferInterval;
     }
 
+    public int getRemoteTranslogExtraKeep() {
+        return remoteTranslogKeepExtraGen;
+    }
+
     /**
      * Returns true iff the remote translog buffer interval setting exists or in other words is explicitly set.
      */
@@ -1311,6 +1326,10 @@ public void setRemoteTranslogUploadBufferInterval(TimeValue remoteTranslogUpload
         this.remoteTranslogUploadBufferInterval = remoteTranslogUploadBufferInterval;
     }
 
+    public void setRemoteTranslogKeepExtraGen(int extraGen) {
+        this.remoteTranslogKeepExtraGen = extraGen;
+    }
+
     /**
      * Returns this interval in which the shards of this index are asynchronously refreshed. {@code -1} means async refresh is disabled.
      */
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
index db85a37b556fc..a305a774f5854 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -431,7 +431,7 @@ public void trimUnreferencedReaders() throws IOException {
         // cleans up remote translog files not referenced in latest uploaded metadata.
         // This enables us to restore translog from the metadata in case of failover or relocation.
         Set generationsToDelete = new HashSet<>();
-        for (long generation = minRemoteGenReferenced - 1; generation >= 0; generation--) {
+        for (long generation = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); generation >= 0; generation--) {
             if (fileTransferTracker.uploaded(Translog.getFilename(generation)) == false) {
                 break;
             }
diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
index 42e0df2dc90c1..3cb65610fab58 100644
--- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
@@ -97,6 +97,7 @@
 import java.util.zip.CheckedInputStream;
 
 import static org.opensearch.common.util.BigArrays.NON_RECYCLING_INSTANCE;
+import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING;
 import static org.opensearch.index.translog.RemoteFsTranslog.TRANSLOG;
 import static org.opensearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder;
 import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy;
@@ -124,6 +125,8 @@ public class RemoteFsTranslogTests extends OpenSearchTestCase {
     private ThreadPool threadPool;
     private final static String METADATA_DIR = "metadata";
     private final static String DATA_DIR = "data";
+
+    AtomicInteger writeCalls = new AtomicInteger();
     BlobStoreRepository repository;
 
     BlobStoreTransferService blobStoreTransferService;
@@ -163,13 +166,13 @@ public void tearDown() throws Exception {
 
     private RemoteFsTranslog create(Path path) throws IOException {
         final String translogUUID = Translog.createEmptyTranslog(path, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
-        return create(path, createRepository(), translogUUID);
+        return create(path, createRepository(), translogUUID, 0);
     }
 
-    private RemoteFsTranslog create(Path path, BlobStoreRepository repository, String translogUUID) throws IOException {
+    private RemoteFsTranslog create(Path path, BlobStoreRepository repository, String translogUUID, int extraGenToKeep) throws IOException {
         this.repository = repository;
         globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
-        final TranslogConfig translogConfig = getTranslogConfig(path);
+        final TranslogConfig translogConfig = getTranslogConfig(path, extraGenToKeep);
         final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(translogConfig.getIndexSettings());
         threadPool = new TestThreadPool(getClass().getName());
         blobStoreTransferService = new BlobStoreTransferService(repository.blobStore(), threadPool);
@@ -185,10 +188,17 @@ private RemoteFsTranslog create(Path path, BlobStoreRepository repository, Strin
             primaryMode::get,
             new RemoteTranslogTransferTracker(shardId, 10)
         );
+    }
 
+    private RemoteFsTranslog create(Path path, BlobStoreRepository repository, String translogUUID) throws IOException {
+        return create(path, repository, translogUUID, 0);
     }
 
     private TranslogConfig getTranslogConfig(final Path path) {
+        return getTranslogConfig(path, 0);
+    }
+
+    private TranslogConfig getTranslogConfig(final Path path, int gensToKeep) {
         final Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT)
             // only randomize between nog age retention and a long one, so failures will have a chance of reproducing
@@ -196,6 +206,7 @@ private TranslogConfig getTranslogConfig(final Path path) {
             .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomIntBetween(-1, 2048) + "b")
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true)
+            .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), gensToKeep)
             .build();
         return getTranslogConfig(path, settings);
     }
@@ -372,6 +383,111 @@ public void testSimpleOperations() throws IOException {
 
     }
 
+    private TranslogConfig getConfig(int gensToKeep) {
+        Path tempDir = createTempDir();
+        final TranslogConfig temp = getTranslogConfig(tempDir, gensToKeep);
+        final TranslogConfig config = new TranslogConfig(
+            temp.getShardId(),
+            temp.getTranslogPath(),
+            temp.getIndexSettings(),
+            temp.getBigArrays(),
+            new ByteSizeValue(1, ByteSizeUnit.KB),
+            ""
+        );
+        return config;
+    }
+
+    private ChannelFactory getChannelFactory() {
+        writeCalls = new AtomicInteger();
+        final ChannelFactory channelFactory = (file, openOption) -> {
+            FileChannel delegate = FileChannel.open(file, openOption);
+            boolean success = false;
+            try {
+                // don't do partial writes for checkpoints we rely on the fact that the bytes are written as an atomic operation
+                final boolean isCkpFile = file.getFileName().toString().endsWith(".ckp");
+
+                final FileChannel channel;
+                if (isCkpFile) {
+                    channel = delegate;
+                } else {
+                    channel = new FilterFileChannel(delegate) {
+
+                        @Override
+                        public int write(ByteBuffer src) throws IOException {
+                            writeCalls.incrementAndGet();
+                            return super.write(src);
+                        }
+                    };
+                }
+                success = true;
+                return channel;
+            } finally {
+                if (success == false) {
+                    IOUtils.closeWhileHandlingException(delegate);
+                }
+            }
+        };
+        return channelFactory;
+    }
+
+    public void testExtraGenToKeep() throws Exception {
+        TranslogConfig config = getConfig(1);
+        ChannelFactory channelFactory = getChannelFactory();
+        final Set persistedSeqNos = new HashSet<>();
+        String translogUUID = Translog.createEmptyTranslog(
+            config.getTranslogPath(),
+            SequenceNumbers.NO_OPS_PERFORMED,
+            shardId,
+            channelFactory,
+            primaryTerm.get()
+        );
+        TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(config.getIndexSettings());
+        ArrayList ops = new ArrayList<>();
+        try (
+            RemoteFsTranslog translog = new RemoteFsTranslog(
+                config,
+                translogUUID,
+                deletionPolicy,
+                () -> SequenceNumbers.NO_OPS_PERFORMED,
+                primaryTerm::get,
+                persistedSeqNos::add,
+                repository,
+                threadPool,
+                () -> Boolean.TRUE,
+                new RemoteTranslogTransferTracker(shardId, 10)
+            ) {
+                @Override
+                ChannelFactory getChannelFactory() {
+                    return channelFactory;
+                }
+            }
+        ) {
+            addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 }));
+
+            addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 1, primaryTerm.get(), new byte[] { 1 }));
+
+            addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 2, primaryTerm.get(), new byte[] { 1 }));
+
+            // expose the new checkpoint (simulating a commit), before we trim the translog
+            translog.setMinSeqNoToKeep(2);
+
+            // Trims from local
+            translog.trimUnreferencedReaders();
+            assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
+
+            addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 3, primaryTerm.get(), new byte[] { 1 }));
+
+            // Trims from remote now
+            translog.trimUnreferencedReaders();
+            assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
+            assertEquals(
+                6,
+                blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size()
+            );
+
+        }
+    }
+
     public void testReadLocation() throws IOException {
         ArrayList ops = new ArrayList<>();
         ArrayList locs = new ArrayList<>();
@@ -619,14 +735,22 @@ public void testSimpleOperationsUpload() throws Exception {
         // this should now trim as tlog-2 files from remote, but not tlog-3 and tlog-4
         addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 }));
         assertEquals(2, translog.stats().estimatedNumberOfOperations());
+        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
 
         translog.setMinSeqNoToKeep(2);
-
-        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
+        // this should now trim as tlog-2 files from remote, but not tlog-3 and tlog-4
         translog.trimUnreferencedReaders();
+        assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable()));
         assertEquals(1, translog.readers.size());
         assertEquals(1, translog.stats().estimatedNumberOfOperations());
-        assertBusy(() -> assertEquals(4, translog.allUploaded().size()));
+        assertBusy(() -> {
+            assertEquals(4, translog.allUploaded().size());
+            assertEquals(
+                4,
+                blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size()
+            );
+        });
+
     }
 
     public void testMetadataFileDeletion() throws Exception {
@@ -1273,49 +1397,10 @@ public void testTranslogWriter() throws IOException {
     }
 
     public void testTranslogWriterCanFlushInAddOrReadCall() throws IOException {
-        Path tempDir = createTempDir();
-        final TranslogConfig temp = getTranslogConfig(tempDir);
-        final TranslogConfig config = new TranslogConfig(
-            temp.getShardId(),
-            temp.getTranslogPath(),
-            temp.getIndexSettings(),
-            temp.getBigArrays(),
-            new ByteSizeValue(1, ByteSizeUnit.KB),
-            ""
-        );
-
+        final TranslogConfig config = getConfig(1);
         final Set persistedSeqNos = new HashSet<>();
-        final AtomicInteger writeCalls = new AtomicInteger();
-
-        final ChannelFactory channelFactory = (file, openOption) -> {
-            FileChannel delegate = FileChannel.open(file, openOption);
-            boolean success = false;
-            try {
-                // don't do partial writes for checkpoints we rely on the fact that the bytes are written as an atomic operation
-                final boolean isCkpFile = file.getFileName().toString().endsWith(".ckp");
-
-                final FileChannel channel;
-                if (isCkpFile) {
-                    channel = delegate;
-                } else {
-                    channel = new FilterFileChannel(delegate) {
-
-                        @Override
-                        public int write(ByteBuffer src) throws IOException {
-                            writeCalls.incrementAndGet();
-                            return super.write(src);
-                        }
-                    };
-                }
-                success = true;
-                return channel;
-            } finally {
-                if (success == false) {
-                    IOUtils.closeWhileHandlingException(delegate);
-                }
-            }
-        };
-
+        writeCalls = new AtomicInteger();
+        final ChannelFactory channelFactory = getChannelFactory();
         String translogUUID = Translog.createEmptyTranslog(
             config.getTranslogPath(),
             SequenceNumbers.NO_OPS_PERFORMED,

From 8f13dee77a7e78833cf90b20607cb4d714032bd8 Mon Sep 17 00:00:00 2001
From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com>
Date: Mon, 23 Oct 2023 14:50:20 +0530
Subject: [PATCH 220/388] Make index and global metadata upload timeout dynamic
 cluster settings (#10814)

* Make index and global metadata upload wait time dynamic

Signed-off-by: Rahul Karajgikar 
---
 CHANGELOG.md                                  |  1 +
 .../common/settings/ClusterSettings.java      |  2 +
 .../remote/RemoteClusterStateService.java     | 47 +++++++++++++++++--
 .../RemoteClusterStateServiceTests.java       | 36 +++++++++++++-
 4 files changed, 80 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 374dd4ab57ee6..9874e7c431b7c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -94,6 +94,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567))
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
+- [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 7ac7da819b215..a0fca4f0a2ff0 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -682,6 +682,8 @@ public void apply(Settings value, Settings current, Settings previous) {
 
                 // Remote cluster state settings
                 RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING,
+                RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING,
+                RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING,
                 RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
                 IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
                 IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index b505b3e2d4c0c..ae4a3fab9852d 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -83,9 +83,23 @@ public class RemoteClusterStateService implements Closeable {
 
     private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class);
 
-    // TODO make this two variable as dynamic setting [issue: #10688]
-    public static final int INDEX_METADATA_UPLOAD_WAIT_MILLIS = 20000;
-    public static final int GLOBAL_METADATA_UPLOAD_WAIT_MILLIS = 20000;
+    public static final TimeValue INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000);
+
+    public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000);
+
+    public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting(
+        "cluster.remote_store.state.index_metadata.upload_timeout",
+        INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting(
+        "cluster.remote_store.state.global_metadata.upload_timeout",
+        GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
 
     public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
         "index-metadata",
@@ -141,6 +155,9 @@ public class RemoteClusterStateService implements Closeable {
     private BlobStoreTransferService blobStoreTransferService;
     private volatile TimeValue slowWriteLoggingThreshold;
 
+    private volatile TimeValue indexMetadataUploadTimeout;
+    private volatile TimeValue globalMetadataUploadTimeout;
+
     private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false);
 
     public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1;
@@ -171,7 +188,11 @@ public RemoteClusterStateService(
         this.relativeTimeNanosSupplier = relativeTimeNanosSupplier;
         this.threadpool = threadPool;
         this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD);
+        this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING);
+        this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING);
         clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold);
+        clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout);
+        clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout);
     }
 
     private BlobStoreTransferService getBlobStoreTransferService() {
@@ -367,7 +388,7 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException
         );
 
         try {
-            if (latch.await(GLOBAL_METADATA_UPLOAD_WAIT_MILLIS, TimeUnit.MILLISECONDS) == false) {
+            if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
                 // TODO: We should add metrics where transfer is timing out. [Issue: #10687]
                 GlobalMetadataTransferException ex = new GlobalMetadataTransferException(
                     String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete")
@@ -422,7 +443,7 @@ private List writeIndexMetadataParallel(ClusterState clus
         }
 
         try {
-            if (latch.await(INDEX_METADATA_UPLOAD_WAIT_MILLIS, TimeUnit.MILLISECONDS) == false) {
+            if (latch.await(getIndexMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
                 IndexMetadataTransferException ex = new IndexMetadataTransferException(
                     String.format(
                         Locale.ROOT,
@@ -621,6 +642,22 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) {
         this.slowWriteLoggingThreshold = slowWriteLoggingThreshold;
     }
 
+    private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) {
+        this.indexMetadataUploadTimeout = newIndexMetadataUploadTimeout;
+    }
+
+    private void setGlobalMetadataUploadTimeout(TimeValue newGlobalMetadataUploadTimeout) {
+        this.globalMetadataUploadTimeout = newGlobalMetadataUploadTimeout;
+    }
+
+    public TimeValue getIndexMetadataUploadTimeout() {
+        return this.indexMetadataUploadTimeout;
+    }
+
+    public TimeValue getGlobalMetadataUploadTimeout() {
+        return this.globalMetadataUploadTimeout;
+    }
+
     static String getManifestFileName(long term, long version, boolean committed) {
         // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____
         return String.join(
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 827735f6bcc0b..4be5fc03c2a6d 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -102,6 +102,7 @@
 public class RemoteClusterStateServiceTests extends OpenSearchTestCase {
 
     private RemoteClusterStateService remoteClusterStateService;
+    private ClusterSettings clusterSettings;
     private Supplier repositoriesServiceSupplier;
     private RepositoriesService repositoriesService;
     private BlobStoreRepository blobStoreRepository;
@@ -132,6 +133,7 @@ public void setup() {
             .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
             .build();
 
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(
             Stream.of(
                 NetworkModule.getNamedXContents().stream(),
@@ -149,7 +151,7 @@ public void setup() {
             "test-node-id",
             repositoriesServiceSupplier,
             settings,
-            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            clusterSettings,
             () -> 0L,
             threadPool
         );
@@ -1053,6 +1055,38 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti
         assertBusy(() -> assertEquals(1, callCount.get()));
     }
 
+    public void testIndexMetadataUploadWaitTimeSetting() {
+        // verify default value
+        assertEquals(
+            RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT,
+            remoteClusterStateService.getIndexMetadataUploadTimeout()
+        );
+
+        // verify update index metadata upload timeout
+        int indexMetadataUploadTimeout = randomIntBetween(1, 10);
+        Settings newSettings = Settings.builder()
+            .put("cluster.remote_store.state.index_metadata.upload_timeout", indexMetadataUploadTimeout + "s")
+            .build();
+        clusterSettings.applySettings(newSettings);
+        assertEquals(indexMetadataUploadTimeout, remoteClusterStateService.getIndexMetadataUploadTimeout().seconds());
+    }
+
+    public void testGlobalMetadataUploadWaitTimeSetting() {
+        // verify default value
+        assertEquals(
+            RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT,
+            remoteClusterStateService.getGlobalMetadataUploadTimeout()
+        );
+
+        // verify update global metadata upload timeout
+        int globalMetadataUploadTimeout = randomIntBetween(1, 10);
+        Settings newSettings = Settings.builder()
+            .put("cluster.remote_store.state.global_metadata.upload_timeout", globalMetadataUploadTimeout + "s")
+            .build();
+        clusterSettings.applySettings(newSettings);
+        assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds());
+    }
+
     private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException {
         mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false);
     }

From d84a00bbcadd41a4631eed3a96691645322ee5c0 Mon Sep 17 00:00:00 2001
From: Sachin Kale 
Date: Mon, 23 Oct 2023 16:59:55 +0530
Subject: [PATCH 221/388] Revert "Validate checksum of each segment file post
 download from remote store (#10119)" (#10847)

This reverts commit 6c022612769e6bc7ed18b9ec0888d6cd0dd415cc.

Signed-off-by: Sachin Kale 
Co-authored-by: Sachin Kale 
---
 .../remotestore/RemoteStoreRestoreIT.java     | 33 +------------
 .../store/RemoteSegmentStoreDirectory.java    |  4 --
 .../org/opensearch/index/store/Store.java     | 47 +------------------
 3 files changed, 3 insertions(+), 81 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
index 212f797180077..7626e3dba6424 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
@@ -10,11 +10,8 @@
 
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse;
-import org.opensearch.action.admin.indices.get.GetIndexRequest;
-import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.health.ClusterHealthStatus;
-import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
@@ -22,12 +19,10 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
-import org.opensearch.test.CorruptionUtils;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Locale;
@@ -35,14 +30,13 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThan;
 
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0)
 public class RemoteStoreRestoreIT extends BaseRemoteStoreRestoreIT {
 
     /**
@@ -467,30 +461,5 @@ public void testRateLimitedRemoteDownloads() throws Exception {
         }
     }
 
-    public void testRestoreCorruptSegmentShouldFail() throws IOException, ExecutionException, InterruptedException {
-        prepareCluster(1, 3, INDEX_NAME, 0, 1);
-        indexData(randomIntBetween(3, 4), true, INDEX_NAME);
-
-        GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest()).get();
-        String indexUUID = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
-
-        logger.info("--> Corrupting segment files in remote segment store");
-        Path path = segmentRepoPath.resolve(indexUUID).resolve("0").resolve("segments").resolve("data");
-        try (Stream dataPath = Files.list(path)) {
-            CorruptionUtils.corruptFile(random(), dataPath.toArray(Path[]::new));
-        }
-
-        logger.info("--> Stop primary");
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME)));
-
-        logger.info("--> Close and restore the index");
-        client().admin()
-            .cluster()
-            .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).waitForCompletion(true), PlainActionFuture.newFuture());
-
-        logger.info("--> Check for index status, should be red due to corruption");
-        ensureRed(INDEX_NAME);
-    }
-
     // TODO: Restore flow - index aliases
 }
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
index 6b43fed3d8930..be1f2341236ab 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
@@ -290,10 +290,6 @@ public void setWrittenByMajor(int writtenByMajor) {
                 );
             }
         }
-
-        public int getWrittenByMajor() {
-            return writtenByMajor;
-        }
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java
index d0cd2635ba672..b822742de6e97 100644
--- a/server/src/main/java/org/opensearch/index/store/Store.java
+++ b/server/src/main/java/org/opensearch/index/store/Store.java
@@ -105,7 +105,6 @@
 import java.io.UncheckedIOException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
-import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -121,7 +120,6 @@
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
-import static java.lang.Character.MAX_RADIX;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.opensearch.index.seqno.SequenceNumbers.LOCAL_CHECKPOINT_KEY;
@@ -977,11 +975,7 @@ public void copyFrom(Directory from, String src, String dest, IOContext context)
             boolean success = false;
             long startTime = System.currentTimeMillis();
             try {
-                if (from instanceof RemoteSegmentStoreDirectory) {
-                    copyFileAndValidateChecksum(from, src, dest, context, fileSize);
-                } else {
-                    super.copyFrom(from, src, dest, context);
-                }
+                super.copyFrom(from, src, dest, context);
                 success = true;
                 afterDownload(fileSize, startTime);
             } finally {
@@ -991,43 +985,6 @@ public void copyFrom(Directory from, String src, String dest, IOContext context)
             }
         }
 
-        private void copyFileAndValidateChecksum(Directory from, String src, String dest, IOContext context, long fileSize)
-            throws IOException {
-            RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = ((RemoteSegmentStoreDirectory) from)
-                .getSegmentsUploadedToRemoteStore()
-                .get(dest);
-            boolean success = false;
-            try (IndexInput is = from.openInput(src, context); IndexOutput os = createOutput(dest, context)) {
-                // Here, we don't need the exact version as LuceneVerifyingIndexOutput does not verify version
-                // It is just used to emit logs when the entire metadata object is provided as parameter. Also,
-                // we can't provide null version as StoreFileMetadata has non-null check on writtenBy field.
-                Version luceneMajorVersion = Version.parse(metadata.getWrittenByMajor() + ".0.0");
-                long checksum = Long.parseLong(metadata.getChecksum());
-                StoreFileMetadata storeFileMetadata = new StoreFileMetadata(
-                    dest,
-                    fileSize,
-                    Long.toString(checksum, MAX_RADIX),
-                    luceneMajorVersion
-                );
-                VerifyingIndexOutput verifyingIndexOutput = new LuceneVerifyingIndexOutput(storeFileMetadata, os);
-                verifyingIndexOutput.copyBytes(is, is.length());
-                verifyingIndexOutput.verify();
-                success = true;
-            } catch (ParseException e) {
-                throw new IOException("Exception while reading version info for segment file from remote store: " + dest, e);
-            } finally {
-                if (success == false) {
-                    // If the exception is thrown after file is created, we clean up the file.
-                    // We ignore the exception as the deletion is best-effort basis and can fail if file does not exist.
-                    try {
-                        deleteFile("Quietly deleting", dest);
-                    } catch (Exception e) {
-                        // Ignore
-                    }
-                }
-            }
-        }
-
         /**
          * Updates the amount of bytes attempted for download
          */
@@ -1519,7 +1476,7 @@ public static boolean isAutogenerated(String name) {
      * Produces a string representation of the given digest value.
      */
     public static String digestToString(long digest) {
-        return Long.toString(digest, MAX_RADIX);
+        return Long.toString(digest, Character.MAX_RADIX);
     }
 
     /**

From 08ffd3586d381fe3526965e8a4570226a9704407 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Oct 2023 11:21:44 -0400
Subject: [PATCH 222/388] Bump org.apache.logging.log4j:log4j-core from 2.20.0
 to 2.21.0 (#10858)

* Bump org.apache.logging.log4j:log4j-core

Bumps org.apache.logging.log4j:log4j-core from 2.20.0 to 2.21.0.

---
updated-dependencies:
- dependency-name: org.apache.logging.log4j:log4j-core
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

* Bumps org.apache.logging.log4j:log4j-core from 2.20.0 to 2.21.0

Signed-off-by: Andriy Redko 

---------

Signed-off-by: dependabot[bot] 
Signed-off-by: Andriy Redko 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
Co-authored-by: Andriy Redko 
---
 CHANGELOG.md                                                 | 3 ++-
 .../src/testKit/thirdPartyAudit/sample_jars/build.gradle     | 3 ++-
 buildSrc/version.properties                                  | 2 +-
 libs/core/licenses/log4j-api-2.20.0.jar.sha1                 | 1 -
 libs/core/licenses/log4j-api-2.21.0.jar.sha1                 | 1 +
 plugins/crypto-kms/licenses/log4j-1.2-api-2.20.0.jar.sha1    | 1 -
 plugins/crypto-kms/licenses/log4j-1.2-api-2.21.0.jar.sha1    | 1 +
 .../licenses/log4j-1.2-api-2.20.0.jar.sha1                   | 1 -
 .../licenses/log4j-1.2-api-2.21.0.jar.sha1                   | 1 +
 plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 -
 plugins/discovery-ec2/licenses/log4j-1.2-api-2.21.0.jar.sha1 | 1 +
 plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 -
 plugins/discovery-gce/licenses/log4j-1.2-api-2.21.0.jar.sha1 | 1 +
 .../repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1    | 1 -
 .../repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1    | 1 +
 .../licenses/log4j-slf4j-impl-2.20.0.jar.sha1                | 1 -
 .../licenses/log4j-slf4j-impl-2.21.0.jar.sha1                | 1 +
 plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1 | 1 -
 plugins/repository-s3/licenses/log4j-1.2-api-2.21.0.jar.sha1 | 1 +
 qa/os/build.gradle                                           | 5 +++++
 server/build.gradle                                          | 4 +++-
 server/licenses/log4j-api-2.20.0.jar.sha1                    | 1 -
 server/licenses/log4j-api-2.21.0.jar.sha1                    | 1 +
 server/licenses/log4j-core-2.20.0.jar.sha1                   | 1 -
 server/licenses/log4j-core-2.21.0.jar.sha1                   | 1 +
 server/licenses/log4j-jul-2.20.0.jar.sha1                    | 1 -
 server/licenses/log4j-jul-2.21.0.jar.sha1                    | 1 +
 27 files changed, 24 insertions(+), 15 deletions(-)
 delete mode 100644 libs/core/licenses/log4j-api-2.20.0.jar.sha1
 create mode 100644 libs/core/licenses/log4j-api-2.21.0.jar.sha1
 delete mode 100644 plugins/crypto-kms/licenses/log4j-1.2-api-2.20.0.jar.sha1
 create mode 100644 plugins/crypto-kms/licenses/log4j-1.2-api-2.21.0.jar.sha1
 delete mode 100644 plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1
 create mode 100644 plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.21.0.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/log4j-1.2-api-2.21.0.jar.sha1
 delete mode 100644 plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1
 create mode 100644 plugins/discovery-gce/licenses/log4j-1.2-api-2.21.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1
 delete mode 100644 plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.21.0.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/log4j-1.2-api-2.21.0.jar.sha1
 delete mode 100644 server/licenses/log4j-api-2.20.0.jar.sha1
 create mode 100644 server/licenses/log4j-api-2.21.0.jar.sha1
 delete mode 100644 server/licenses/log4j-core-2.20.0.jar.sha1
 create mode 100644 server/licenses/log4j-core-2.21.0.jar.sha1
 delete mode 100644 server/licenses/log4j-jul-2.20.0.jar.sha1
 create mode 100644 server/licenses/log4j-jul-2.21.0.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9874e7c431b7c..8c4563a216974 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -107,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.0 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -130,4 +131,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file
diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
index cb8050d1718c4..74c88e0961c9c 100644
--- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
+++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
@@ -15,8 +15,9 @@ plugins {
 repositories {
   mavenCentral()
 }
+
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.20.0"
+  implementation "org.apache.logging.log4j:log4j-core:2.21.0"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index a5171aa582a86..96d398c35851d 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -14,7 +14,7 @@ jackson_databind  = 2.15.2
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
-log4j             = 2.20.0
+log4j             = 2.21.0
 slf4j             = 1.7.36
 asm               = 9.6
 jettison          = 1.5.4
diff --git a/libs/core/licenses/log4j-api-2.20.0.jar.sha1 b/libs/core/licenses/log4j-api-2.20.0.jar.sha1
deleted file mode 100644
index 37154d9861ac0..0000000000000
--- a/libs/core/licenses/log4j-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1fe6082e660daf07c689a89c94dc0f49c26b44bb
\ No newline at end of file
diff --git a/libs/core/licenses/log4j-api-2.21.0.jar.sha1 b/libs/core/licenses/log4j-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..51446052594aa
--- /dev/null
+++ b/libs/core/licenses/log4j-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+760192f2b69eacf4a4afc78e5a1d7a8de054fcbd
\ No newline at end of file
diff --git a/plugins/crypto-kms/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/crypto-kms/licenses/log4j-1.2-api-2.20.0.jar.sha1
deleted file mode 100644
index 9829576d38ce0..0000000000000
--- a/plugins/crypto-kms/licenses/log4j-1.2-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-689151374756cb809cb029f2501015bdc7733179
\ No newline at end of file
diff --git a/plugins/crypto-kms/licenses/log4j-1.2-api-2.21.0.jar.sha1 b/plugins/crypto-kms/licenses/log4j-1.2-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..39d9177cb2fac
--- /dev/null
+++ b/plugins/crypto-kms/licenses/log4j-1.2-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1
deleted file mode 100644
index 9829576d38ce0..0000000000000
--- a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-689151374756cb809cb029f2501015bdc7733179
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.21.0.jar.sha1 b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..39d9177cb2fac
--- /dev/null
+++ b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1
deleted file mode 100644
index 9829576d38ce0..0000000000000
--- a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-689151374756cb809cb029f2501015bdc7733179
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.21.0.jar.sha1 b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..39d9177cb2fac
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1
deleted file mode 100644
index 9829576d38ce0..0000000000000
--- a/plugins/discovery-gce/licenses/log4j-1.2-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-689151374756cb809cb029f2501015bdc7733179
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/log4j-1.2-api-2.21.0.jar.sha1 b/plugins/discovery-gce/licenses/log4j-1.2-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..39d9177cb2fac
--- /dev/null
+++ b/plugins/discovery-gce/licenses/log4j-1.2-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1
deleted file mode 100644
index 9829576d38ce0..0000000000000
--- a/plugins/repository-gcs/licenses/log4j-1.2-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-689151374756cb809cb029f2501015bdc7733179
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1 b/plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..39d9177cb2fac
--- /dev/null
+++ b/plugins/repository-gcs/licenses/log4j-1.2-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1
deleted file mode 100644
index 800a4aa87ba0e..0000000000000
--- a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7ab4f082fd162f60afcaf2b8744a3d959feab3e8
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..0e22f98daa61c
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.21.0.jar.sha1
@@ -0,0 +1 @@
+911fdb5b1a1df36719c579ecc6f2957b88bce1ab
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1 b/plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1
deleted file mode 100644
index 9829576d38ce0..0000000000000
--- a/plugins/repository-s3/licenses/log4j-1.2-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-689151374756cb809cb029f2501015bdc7733179
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/log4j-1.2-api-2.21.0.jar.sha1 b/plugins/repository-s3/licenses/log4j-1.2-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..39d9177cb2fac
--- /dev/null
+++ b/plugins/repository-s3/licenses/log4j-1.2-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+12bad3819a9570807f3c97315930699584c12152
\ No newline at end of file
diff --git a/qa/os/build.gradle b/qa/os/build.gradle
index 66c6525439dac..082ed5277575a 100644
--- a/qa/os/build.gradle
+++ b/qa/os/build.gradle
@@ -70,6 +70,11 @@ tasks.dependenciesInfo.enabled = false
 
 tasks.thirdPartyAudit.ignoreMissingClasses()
 
+tasks.thirdPartyAudit.ignoreViolations(
+  'org.apache.logging.log4j.core.util.internal.UnsafeUtil',
+  'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1'
+)
+
 tasks.register('destructivePackagingTest') {
   dependsOn 'destructiveDistroTest'
 }
diff --git a/server/build.gradle b/server/build.gradle
index f6db3d53a0dcc..c56f9d5aa288f 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -364,7 +364,9 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
             'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
             'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
-            'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+            'com.google.protobuf.UnsafeUtil$MemoryAccessor',
+            'org.apache.logging.log4j.core.util.internal.UnsafeUtil',
+            'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1'
     )
 }
 
diff --git a/server/licenses/log4j-api-2.20.0.jar.sha1 b/server/licenses/log4j-api-2.20.0.jar.sha1
deleted file mode 100644
index 37154d9861ac0..0000000000000
--- a/server/licenses/log4j-api-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1fe6082e660daf07c689a89c94dc0f49c26b44bb
\ No newline at end of file
diff --git a/server/licenses/log4j-api-2.21.0.jar.sha1 b/server/licenses/log4j-api-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..51446052594aa
--- /dev/null
+++ b/server/licenses/log4j-api-2.21.0.jar.sha1
@@ -0,0 +1 @@
+760192f2b69eacf4a4afc78e5a1d7a8de054fcbd
\ No newline at end of file
diff --git a/server/licenses/log4j-core-2.20.0.jar.sha1 b/server/licenses/log4j-core-2.20.0.jar.sha1
deleted file mode 100644
index 49c972626563b..0000000000000
--- a/server/licenses/log4j-core-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb2a9a47b1396e00b5eee1264296729a70565cc0
\ No newline at end of file
diff --git a/server/licenses/log4j-core-2.21.0.jar.sha1 b/server/licenses/log4j-core-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..c88e6f7a25ca9
--- /dev/null
+++ b/server/licenses/log4j-core-2.21.0.jar.sha1
@@ -0,0 +1 @@
+122e1a9e0603cc9eae07b0846a6ff01f2454bc49
\ No newline at end of file
diff --git a/server/licenses/log4j-jul-2.20.0.jar.sha1 b/server/licenses/log4j-jul-2.20.0.jar.sha1
deleted file mode 100644
index a456651e4569e..0000000000000
--- a/server/licenses/log4j-jul-2.20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8170e6118eac1ab332046c179718a0f107f688e1
\ No newline at end of file
diff --git a/server/licenses/log4j-jul-2.21.0.jar.sha1 b/server/licenses/log4j-jul-2.21.0.jar.sha1
new file mode 100644
index 0000000000000..480010840abca
--- /dev/null
+++ b/server/licenses/log4j-jul-2.21.0.jar.sha1
@@ -0,0 +1 @@
+f0da61113f4a47654677e6a98b1e13ca7de2483d
\ No newline at end of file

From da851244a273d6fbfc42eb115f2dcfca1c3cc728 Mon Sep 17 00:00:00 2001
From: David Z <38449481+dzane17@users.noreply.github.com>
Date: Mon, 23 Oct 2023 11:01:47 -0700
Subject: [PATCH 223/388] Update per request latency check to V_2_12_0 (#10865)

Signed-off-by: David Zane 
---
 .../main/java/org/opensearch/action/search/SearchRequest.java | 4 ++--
 .../java/org/opensearch/action/search/SearchResponse.java     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
index 9e50213eab5f9..fb026dae630b7 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
@@ -256,7 +256,7 @@ public SearchRequest(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_2_7_0)) {
             pipeline = in.readOptionalString();
         }
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             phaseTook = in.readOptionalBoolean();
         }
     }
@@ -290,7 +290,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_7_0)) {
             out.writeOptionalString(pipeline);
         }
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalBoolean(phaseTook);
         }
     }
diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponse.java b/server/src/main/java/org/opensearch/action/search/SearchResponse.java
index 91f0dc0737637..96d07982d03db 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchResponse.java
@@ -116,7 +116,7 @@ public SearchResponse(StreamInput in) throws IOException {
         clusters = new Clusters(in);
         scrollId = in.readOptionalString();
         tookInMillis = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             phaseTook = in.readOptionalWriteable(PhaseTook::new);
         } else {
             phaseTook = null;
@@ -557,7 +557,7 @@ public void writeTo(StreamOutput out) throws IOException {
         clusters.writeTo(out);
         out.writeOptionalString(scrollId);
         out.writeVLong(tookInMillis);
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(phaseTook);
         }
         out.writeVInt(skippedShards);

From 8b2173910f754a48773b3283e1a511cbc1a9db78 Mon Sep 17 00:00:00 2001
From: Poojita Raj 
Date: Mon, 23 Oct 2023 16:08:04 -0700
Subject: [PATCH 224/388] =?UTF-8?q?Add=20cluster=20setting=20cluster.restr?=
 =?UTF-8?q?ict.index.replication=5Ftype=20t=E2=80=A6=20(#10866)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add cluster setting CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING to restrict setting of index setting replication type

Signed-off-by: Poojita Raj 

* Add Changelog entry

Signed-off-by: Poojita Raj 

* refactoring

Signed-off-by: Poojita Raj 

---------

Signed-off-by: Poojita Raj 
---
 CHANGELOG.md                                  |  3 ++-
 .../SegmentReplicationClusterSettingIT.java   | 27 +++++++++++++++++++
 .../metadata/MetadataCreateIndexService.java  | 19 +++++++++++++
 .../common/settings/ClusterSettings.java      |  3 ++-
 .../opensearch/indices/IndicesService.java    | 11 ++++++++
 .../MetadataCreateIndexServiceTests.java      | 16 +++++++++--
 6 files changed, 75 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c4563a216974..0b0d9720ad208 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -95,6 +95,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
+- Added cluster setting cluster.restrict.index.replication_type to restrict setting of index setting replication type ([#10866](https://github.com/opensearch-project/OpenSearch/pull/10866))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
@@ -131,4 +132,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java
index a82fd8d845709..186a5ce39f131 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java
@@ -19,6 +19,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
+import static org.opensearch.indices.IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
@@ -123,4 +124,30 @@ public void testIndexReplicationSettingOverridesDocRepClusterSetting() throws Ex
         assertEquals(indicesService.indexService(anotherIndex).getIndexSettings().isSegRepEnabled(), false);
     }
 
+    public void testIndexReplicationTypeWhenRestrictSettingTrue() {
+        testRestrictIndexReplicationTypeSetting(true, randomFrom(ReplicationType.values()));
+    }
+
+    public void testIndexReplicationTypeWhenRestrictSettingFalse() {
+        testRestrictIndexReplicationTypeSetting(false, randomFrom(ReplicationType.values()));
+    }
+
+    private void testRestrictIndexReplicationTypeSetting(boolean setRestrict, ReplicationType replicationType) {
+        String expectedExceptionMsg =
+            "Validation Failed: 1: index setting [index.replication.type] is not allowed to be set as [cluster.restrict.index.replication_type=true];";
+        String clusterManagerName = internalCluster().startNode(
+            Settings.builder().put(CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey(), setRestrict).build()
+        );
+        internalCluster().startDataOnlyNodes(1);
+
+        // Test create index fails
+        Settings indexSettings = Settings.builder().put(indexSettings()).put(SETTING_REPLICATION_TYPE, replicationType).build();
+        if (setRestrict) {
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME, indexSettings));
+            assertEquals(expectedExceptionMsg, exception.getMessage());
+        } else {
+            createIndex(INDEX_NAME, indexSettings);
+        }
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 8d76a39712ee3..78a22fe11f072 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -1252,6 +1252,7 @@ List getIndexSettingsValidationErrors(
         if (forbidPrivateIndexSettings) {
             validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings));
         }
+        validateIndexReplicationTypeSettings(settings, clusterService.getClusterSettings()).ifPresent(validationErrors::add);
         if (indexName.isEmpty() || indexName.get().charAt(0) != '.') {
             // Apply aware replica balance validation only to non system indices
             int replicaCount = settings.getAsInt(
@@ -1306,6 +1307,24 @@ private static List validateIndexCustomPath(Settings settings, @Nullable
         return validationErrors;
     }
 
+    /**
+     * Validates {@code index.replication.type} is not set if {@code cluster.restrict.index.replication_type} is set to true.
+     *
+     * @param requestSettings settings passed in during index create request
+     * @param clusterSettings cluster setting
+     */
+    private static Optional validateIndexReplicationTypeSettings(Settings requestSettings, ClusterSettings clusterSettings) {
+        if (requestSettings.hasValue(SETTING_REPLICATION_TYPE)
+            && clusterSettings.get(IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING)) {
+            return Optional.of(
+                "index setting [index.replication.type] is not allowed to be set as ["
+                    + IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey()
+                    + "=true]"
+            );
+        }
+        return Optional.empty();
+    }
+
     /**
      * Validates the settings and mappings for shrinking an index.
      *
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index a0fca4f0a2ff0..c2c6effc3336f 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -690,7 +690,8 @@ public void apply(Settings value, Settings current, Settings previous) {
                 AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
-                CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT
+                CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
+                IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index 50c551c2be29b..36abc77893d81 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -299,6 +299,17 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Final
     );
 
+    /**
+     * This setting is used to restrict creation of index where the 'index.replication.type' index setting is set.
+     * If disabled, the replication type can be specified.
+     */
+    public static final Setting CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING = Setting.boolSetting(
+        "cluster.restrict.index.replication_type",
+        false,
+        Property.NodeScope,
+        Property.Final
+    );
+
     /**
      * The node's settings.
      */
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index e40826915c848..cace66d8c6d9e 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -139,6 +139,7 @@
 import static org.opensearch.indices.IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
+import static org.opensearch.indices.IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING;
 import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService;
 import static org.opensearch.node.Node.NODE_ATTRIBUTES;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
@@ -1177,6 +1178,8 @@ public void testvalidateIndexSettings() {
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b")
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e")
             .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true)
+            .put(CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey(), true)
+            .put(SETTING_REPLICATION_TYPE, randomFrom(ReplicationType.values()))
             .build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         when(clusterService.getSettings()).thenReturn(settings);
@@ -1200,8 +1203,12 @@ public void testvalidateIndexSettings() {
         );
 
         List validationErrors = checkerService.getIndexSettingsValidationErrors(settings, false, Optional.empty());
-        assertThat(validationErrors.size(), is(1));
-        assertThat(validationErrors.get(0), is("expected total copies needs to be a multiple of total awareness attributes [3]"));
+        assertThat(validationErrors.size(), is(2));
+        assertThat(
+            validationErrors.get(0),
+            is("index setting [index.replication.type] is not allowed to be set as [cluster.restrict.index.replication_type=true]")
+        );
+        assertThat(validationErrors.get(1), is("expected total copies needs to be a multiple of total awareness attributes [3]"));
 
         settings = Settings.builder()
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack")
@@ -1209,8 +1216,13 @@ public void testvalidateIndexSettings() {
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e")
             .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true)
             .put(SETTING_NUMBER_OF_REPLICAS, 2)
+            .put(CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey(), false)
+            .put(SETTING_REPLICATION_TYPE, randomFrom(ReplicationType.values()))
             .build();
 
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+
         validationErrors = checkerService.getIndexSettingsValidationErrors(settings, false, Optional.empty());
         assertThat(validationErrors.size(), is(0));
 

From 5a288ed3ecf06a048eba6028c445aa4e642d39c5 Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Tue, 24 Oct 2023 09:02:48 +0530
Subject: [PATCH 225/388] Changing version check to 2.12 for remote cluster
 state (#10844)

Signed-off-by: Dhwanil Patel 
---
 .../opensearch/gateway/remote/ClusterMetadataManifest.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
index 97b37d9532f85..4725f40076ce2 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
@@ -262,7 +262,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
         this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
         this.previousClusterUUID = in.readString();
         this.clusterUUIDCommitted = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             this.codecVersion = in.readInt();
             this.globalMetadataFileName = in.readString();
         } else {
@@ -316,7 +316,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(indices);
         out.writeString(previousClusterUUID);
         out.writeBoolean(clusterUUIDCommitted);
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeInt(codecVersion);
             out.writeString(globalMetadataFileName);
         }

From a2b5f0e3a9515f101653afc88dd98e48015f6346 Mon Sep 17 00:00:00 2001
From: Shivansh Arora <31575408+shiv0408@users.noreply.github.com>
Date: Tue, 24 Oct 2023 13:58:32 +0530
Subject: [PATCH 226/388] Created new urgent priority threadpool for remote
 cluster state uploads (#10685)

* Created new Urgent WritePriority and a new threadpools and S3Client for this priority which is being used to upload cluster state

Signed-off-by: Shivansh Arora 
---
 .../s3/S3BlobStoreRepositoryTests.java        |  2 +-
 .../s3/AmazonAsyncS3Reference.java            |  1 +
 .../s3/AmazonAsyncS3WithCredentials.java      | 10 ++-
 .../repositories/s3/S3AsyncService.java       | 17 +++-
 .../repositories/s3/S3BlobContainer.java      | 11 ++-
 .../repositories/s3/S3BlobStore.java          |  5 +-
 .../repositories/s3/S3Repository.java         |  6 ++
 .../repositories/s3/S3RepositoryPlugin.java   | 22 ++++-
 .../s3/async/AsyncPartsHandler.java           | 15 +++-
 .../s3/async/AsyncTransferManager.java        | 21 ++++-
 .../s3/RepositoryCredentialsTests.java        |  2 +-
 .../repositories/s3/S3AsyncServiceTests.java  |  8 +-
 .../s3/S3BlobContainerMockClientTests.java    |  5 +-
 .../s3/S3BlobContainerRetriesTests.java       |  2 +
 .../s3/S3BlobStoreContainerTests.java         |  8 +-
 .../repositories/s3/S3RepositoryTests.java    |  1 +
 .../s3/async/AsyncTransferManagerTests.java   |  1 +
 .../blobstore/stream/write/WritePriority.java |  3 +-
 .../remote/RemoteClusterStateService.java     |  4 +-
 .../blobstore/ChecksumBlobStoreFormat.java    | 45 ++++++++--
 .../RemoteClusterStateServiceTests.java       |  2 +-
 .../snapshots/BlobStoreFormatTests.java       | 87 +++++++++++++++----
 22 files changed, 224 insertions(+), 54 deletions(-)

diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index 4df30bfd2169e..da2c6e8c1b0ee 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -249,7 +249,7 @@ protected S3Repository createRepository(
             ClusterService clusterService,
             RecoverySettings recoverySettings
         ) {
-            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, false) {
+            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, null, false) {
 
                 @Override
                 public BlobStore blobStore() {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java
index 0b5fcb6df280e..45170ea1ad209 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3Reference.java
@@ -29,6 +29,7 @@ public class AmazonAsyncS3Reference extends RefCountedReleasable {
             client.client().close();
             client.priorityClient().close();
+            client.urgentClient().close();
             AwsCredentialsProvider credentials = client.credentials();
             if (credentials instanceof Closeable) {
                 try {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java
index fa2db83729d25..f8a313b55d945 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/AmazonAsyncS3WithCredentials.java
@@ -19,16 +19,19 @@
 final class AmazonAsyncS3WithCredentials {
     private final S3AsyncClient client;
     private final S3AsyncClient priorityClient;
+    private final S3AsyncClient urgentClient;
     private final AwsCredentialsProvider credentials;
 
     private AmazonAsyncS3WithCredentials(
         final S3AsyncClient client,
         final S3AsyncClient priorityClient,
+        final S3AsyncClient urgentClient,
         @Nullable final AwsCredentialsProvider credentials
     ) {
         this.client = client;
         this.credentials = credentials;
         this.priorityClient = priorityClient;
+        this.urgentClient = urgentClient;
     }
 
     S3AsyncClient client() {
@@ -39,6 +42,10 @@ S3AsyncClient priorityClient() {
         return priorityClient;
     }
 
+    S3AsyncClient urgentClient() {
+        return urgentClient;
+    }
+
     AwsCredentialsProvider credentials() {
         return credentials;
     }
@@ -46,8 +53,9 @@ AwsCredentialsProvider credentials() {
     static AmazonAsyncS3WithCredentials create(
         final S3AsyncClient client,
         final S3AsyncClient priorityClient,
+        final S3AsyncClient urgentClient,
         @Nullable final AwsCredentialsProvider credentials
     ) {
-        return new AmazonAsyncS3WithCredentials(client, priorityClient, credentials);
+        return new AmazonAsyncS3WithCredentials(client, priorityClient, urgentClient, credentials);
     }
 }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java
index 08215ebdd45e0..262304029a0d3 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java
@@ -103,6 +103,7 @@ public synchronized void refreshAndClearCache(Map clie
      */
     public AmazonAsyncS3Reference client(
         RepositoryMetadata repositoryMetadata,
+        AsyncExecutorContainer urgentExecutorBuilder,
         AsyncExecutorContainer priorityExecutorBuilder,
         AsyncExecutorContainer normalExecutorBuilder
     ) {
@@ -119,7 +120,7 @@ public AmazonAsyncS3Reference client(
                 return existing;
             }
             final AmazonAsyncS3Reference clientReference = new AmazonAsyncS3Reference(
-                buildClient(clientSettings, priorityExecutorBuilder, normalExecutorBuilder)
+                buildClient(clientSettings, urgentExecutorBuilder, priorityExecutorBuilder, normalExecutorBuilder)
             );
             clientReference.incRef();
             clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientSettings, clientReference).immutableMap();
@@ -165,6 +166,7 @@ S3ClientSettings settings(RepositoryMetadata repositoryMetadata) {
     // proxy for testing
     synchronized AmazonAsyncS3WithCredentials buildClient(
         final S3ClientSettings clientSettings,
+        AsyncExecutorContainer urgentExecutorBuilder,
         AsyncExecutorContainer priorityExecutorBuilder,
         AsyncExecutorContainer normalExecutorBuilder
     ) {
@@ -195,6 +197,17 @@ synchronized AmazonAsyncS3WithCredentials buildClient(
             builder.forcePathStyle(true);
         }
 
+        builder.httpClient(buildHttpClient(clientSettings, urgentExecutorBuilder.getAsyncTransferEventLoopGroup()));
+        builder.asyncConfiguration(
+            ClientAsyncConfiguration.builder()
+                .advancedOption(
+                    SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR,
+                    urgentExecutorBuilder.getFutureCompletionExecutor()
+                )
+                .build()
+        );
+        final S3AsyncClient urgentClient = SocketAccess.doPrivileged(builder::build);
+
         builder.httpClient(buildHttpClient(clientSettings, priorityExecutorBuilder.getAsyncTransferEventLoopGroup()));
         builder.asyncConfiguration(
             ClientAsyncConfiguration.builder()
@@ -217,7 +230,7 @@ synchronized AmazonAsyncS3WithCredentials buildClient(
         );
         final S3AsyncClient client = SocketAccess.doPrivileged(builder::build);
 
-        return AmazonAsyncS3WithCredentials.create(client, priorityClient, credentials);
+        return AmazonAsyncS3WithCredentials.create(client, priorityClient, urgentClient, credentials);
     }
 
     static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSettings clientSettings) {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
index 24aee99242957..c1180aab0e0c7 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
@@ -195,9 +195,14 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener comp
             StreamContext streamContext = SocketAccess.doPrivileged(() -> writeContext.getStreamProvider(partSize));
             try (AmazonAsyncS3Reference amazonS3Reference = SocketAccess.doPrivileged(blobStore::asyncClientReference)) {
 
-                S3AsyncClient s3AsyncClient = writeContext.getWritePriority() == WritePriority.HIGH
-                    ? amazonS3Reference.get().priorityClient()
-                    : amazonS3Reference.get().client();
+                S3AsyncClient s3AsyncClient;
+                if (writeContext.getWritePriority() == WritePriority.URGENT) {
+                    s3AsyncClient = amazonS3Reference.get().urgentClient();
+                } else if (writeContext.getWritePriority() == WritePriority.HIGH) {
+                    s3AsyncClient = amazonS3Reference.get().priorityClient();
+                } else {
+                    s3AsyncClient = amazonS3Reference.get().client();
+                }
                 CompletableFuture completableFuture = blobStore.getAsyncTransferManager()
                     .uploadObject(s3AsyncClient, uploadRequest, streamContext, blobStore.getStatsMetricPublisher());
                 completableFuture.whenComplete((response, throwable) -> {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
index f568d871dd31a..e8e043357e126 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
@@ -84,6 +84,7 @@ class S3BlobStore implements BlobStore {
     private final StatsMetricPublisher statsMetricPublisher = new StatsMetricPublisher();
 
     private final AsyncTransferManager asyncTransferManager;
+    private final AsyncExecutorContainer urgentExecutorBuilder;
     private final AsyncExecutorContainer priorityExecutorBuilder;
     private final AsyncExecutorContainer normalExecutorBuilder;
     private final boolean multipartUploadEnabled;
@@ -100,6 +101,7 @@ class S3BlobStore implements BlobStore {
         int bulkDeletesSize,
         RepositoryMetadata repositoryMetadata,
         AsyncTransferManager asyncTransferManager,
+        AsyncExecutorContainer urgentExecutorBuilder,
         AsyncExecutorContainer priorityExecutorBuilder,
         AsyncExecutorContainer normalExecutorBuilder
     ) {
@@ -116,6 +118,7 @@ class S3BlobStore implements BlobStore {
         this.asyncTransferManager = asyncTransferManager;
         this.normalExecutorBuilder = normalExecutorBuilder;
         this.priorityExecutorBuilder = priorityExecutorBuilder;
+        this.urgentExecutorBuilder = urgentExecutorBuilder;
     }
 
     @Override
@@ -139,7 +142,7 @@ public AmazonS3Reference clientReference() {
     }
 
     public AmazonAsyncS3Reference asyncClientReference() {
-        return s3AsyncService.client(repositoryMetadata, priorityExecutorBuilder, normalExecutorBuilder);
+        return s3AsyncService.client(repositoryMetadata, urgentExecutorBuilder, priorityExecutorBuilder, normalExecutorBuilder);
     }
 
     int getMaxRetries() {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
index aaf5b79891cdc..728a99b1220a6 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
@@ -234,6 +234,7 @@ class S3Repository extends MeteredBlobStoreRepository {
     private final AsyncTransferManager asyncUploadUtils;
     private final S3AsyncService s3AsyncService;
     private final boolean multipartUploadEnabled;
+    private final AsyncExecutorContainer urgentExecutorBuilder;
     private final AsyncExecutorContainer priorityExecutorBuilder;
     private final AsyncExecutorContainer normalExecutorBuilder;
     private final Path pluginConfigPath;
@@ -248,6 +249,7 @@ class S3Repository extends MeteredBlobStoreRepository {
         final ClusterService clusterService,
         final RecoverySettings recoverySettings,
         final AsyncTransferManager asyncUploadUtils,
+        final AsyncExecutorContainer urgentExecutorBuilder,
         final AsyncExecutorContainer priorityExecutorBuilder,
         final AsyncExecutorContainer normalExecutorBuilder,
         final S3AsyncService s3AsyncService,
@@ -260,6 +262,7 @@ class S3Repository extends MeteredBlobStoreRepository {
             clusterService,
             recoverySettings,
             asyncUploadUtils,
+            urgentExecutorBuilder,
             priorityExecutorBuilder,
             normalExecutorBuilder,
             s3AsyncService,
@@ -278,6 +281,7 @@ class S3Repository extends MeteredBlobStoreRepository {
         final ClusterService clusterService,
         final RecoverySettings recoverySettings,
         final AsyncTransferManager asyncUploadUtils,
+        final AsyncExecutorContainer urgentExecutorBuilder,
         final AsyncExecutorContainer priorityExecutorBuilder,
         final AsyncExecutorContainer normalExecutorBuilder,
         final S3AsyncService s3AsyncService,
@@ -290,6 +294,7 @@ class S3Repository extends MeteredBlobStoreRepository {
         this.multipartUploadEnabled = multipartUploadEnabled;
         this.pluginConfigPath = pluginConfigPath;
         this.asyncUploadUtils = asyncUploadUtils;
+        this.urgentExecutorBuilder = urgentExecutorBuilder;
         this.priorityExecutorBuilder = priorityExecutorBuilder;
         this.normalExecutorBuilder = normalExecutorBuilder;
 
@@ -352,6 +357,7 @@ protected S3BlobStore createBlobStore() {
             bulkDeletesSize,
             metadata,
             asyncUploadUtils,
+            urgentExecutorBuilder,
             priorityExecutorBuilder,
             normalExecutorBuilder
         );
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
index c6450e49d08e2..9ed232464d080 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
@@ -75,6 +75,9 @@
  * A plugin to add a repository type that writes to and from the AWS S3.
  */
 public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin {
+
+    private static final String URGENT_FUTURE_COMPLETION = "urgent_future_completion";
+    private static final String URGENT_STREAM_READER = "urgent_stream_reader";
     private static final String PRIORITY_FUTURE_COMPLETION = "priority_future_completion";
     private static final String PRIORITY_STREAM_READER = "priority_stream_reader";
     private static final String FUTURE_COMPLETION = "future_completion";
@@ -85,6 +88,7 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
 
     private final Path configPath;
 
+    private AsyncExecutorContainer urgentExecutorBuilder;
     private AsyncExecutorContainer priorityExecutorBuilder;
     private AsyncExecutorContainer normalExecutorBuilder;
 
@@ -96,6 +100,10 @@ public S3RepositoryPlugin(final Settings settings, final Path configPath) {
     public List> getExecutorBuilders(Settings settings) {
         List> executorBuilders = new ArrayList<>();
         int halfProcMaxAt5 = halfAllocatedProcessorsMaxFive(allocatedProcessors(settings));
+        executorBuilders.add(
+            new FixedExecutorBuilder(settings, URGENT_FUTURE_COMPLETION, urgentPoolCount(settings), 10_000, URGENT_FUTURE_COMPLETION)
+        );
+        executorBuilders.add(new ScalingExecutorBuilder(URGENT_STREAM_READER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
         executorBuilders.add(
             new FixedExecutorBuilder(settings, PRIORITY_FUTURE_COMPLETION, priorityPoolCount(settings), 10_000, PRIORITY_FUTURE_COMPLETION)
         );
@@ -128,6 +136,10 @@ private static int allocatedProcessors(Settings settings) {
         return OpenSearchExecutors.allocatedProcessors(settings);
     }
 
+    private static int urgentPoolCount(Settings settings) {
+        return boundedBy((allocatedProcessors(settings) + 7) / 8, 1, 2);
+    }
+
     private static int priorityPoolCount(Settings settings) {
         return boundedBy((allocatedProcessors(settings) + 1) / 2, 2, 4);
     }
@@ -150,8 +162,14 @@ public Collection createComponents(
         final IndexNameExpressionResolver expressionResolver,
         final Supplier repositoriesServiceSupplier
     ) {
+        int urgentEventLoopThreads = urgentPoolCount(clusterService.getSettings());
         int priorityEventLoopThreads = priorityPoolCount(clusterService.getSettings());
         int normalEventLoopThreads = normalPoolCount(clusterService.getSettings());
+        this.urgentExecutorBuilder = new AsyncExecutorContainer(
+            threadPool.executor(URGENT_FUTURE_COMPLETION),
+            threadPool.executor(URGENT_STREAM_READER),
+            new AsyncTransferEventLoopGroup(urgentEventLoopThreads)
+        );
         this.priorityExecutorBuilder = new AsyncExecutorContainer(
             threadPool.executor(PRIORITY_FUTURE_COMPLETION),
             threadPool.executor(PRIORITY_STREAM_READER),
@@ -176,7 +194,8 @@ protected S3Repository createRepository(
         AsyncTransferManager asyncUploadUtils = new AsyncTransferManager(
             S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.get(clusterService.getSettings()).getBytes(),
             normalExecutorBuilder.getStreamReader(),
-            priorityExecutorBuilder.getStreamReader()
+            priorityExecutorBuilder.getStreamReader(),
+            urgentExecutorBuilder.getStreamReader()
         );
         return new S3Repository(
             metadata,
@@ -185,6 +204,7 @@ protected S3Repository createRepository(
             clusterService,
             recoverySettings,
             asyncUploadUtils,
+            urgentExecutorBuilder,
             priorityExecutorBuilder,
             normalExecutorBuilder,
             s3AsyncService,
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
index 6007d9f9c8a1c..933ee6dc29513 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
@@ -48,6 +48,7 @@ public class AsyncPartsHandler {
      * @param s3AsyncClient S3 client to use for upload
      * @param executorService Thread pool for regular upload
      * @param priorityExecutorService Thread pool for priority uploads
+     * @param urgentExecutorService Thread pool for urgent uploads
      * @param uploadRequest request for upload
      * @param streamContext Stream context used in supplying individual file parts
      * @param uploadId Upload Id against which multi-part is being performed
@@ -60,6 +61,7 @@ public static List> uploadParts(
         S3AsyncClient s3AsyncClient,
         ExecutorService executorService,
         ExecutorService priorityExecutorService,
+        ExecutorService urgentExecutorService,
         UploadRequest uploadRequest,
         StreamContext streamContext,
         String uploadId,
@@ -83,6 +85,7 @@ public static List> uploadParts(
                 s3AsyncClient,
                 executorService,
                 priorityExecutorService,
+                urgentExecutorService,
                 completedParts,
                 inputStreamContainers,
                 futures,
@@ -129,6 +132,7 @@ private static void uploadPart(
         S3AsyncClient s3AsyncClient,
         ExecutorService executorService,
         ExecutorService priorityExecutorService,
+        ExecutorService urgentExecutorService,
         AtomicReferenceArray completedParts,
         AtomicReferenceArray inputStreamContainers,
         List> futures,
@@ -138,9 +142,14 @@ private static void uploadPart(
     ) {
         Integer partNumber = uploadPartRequest.partNumber();
 
-        ExecutorService streamReadExecutor = uploadRequest.getWritePriority() == WritePriority.HIGH
-            ? priorityExecutorService
-            : executorService;
+        ExecutorService streamReadExecutor;
+        if (uploadRequest.getWritePriority() == WritePriority.URGENT) {
+            streamReadExecutor = urgentExecutorService;
+        } else if (uploadRequest.getWritePriority() == WritePriority.HIGH) {
+            streamReadExecutor = priorityExecutorService;
+        } else {
+            streamReadExecutor = executorService;
+        }
         // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
         // data can be retried instead of retrying whole file by the application.
         InputStream inputStream = new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1));
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
index a52745e33073e..4f1ab9764702e 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
@@ -61,6 +61,7 @@ public final class AsyncTransferManager {
     private static final Logger log = LogManager.getLogger(AsyncTransferManager.class);
     private final ExecutorService executorService;
     private final ExecutorService priorityExecutorService;
+    private final ExecutorService urgentExecutorService;
     private final long minimumPartSize;
 
     /**
@@ -75,10 +76,16 @@ public final class AsyncTransferManager {
      * @param executorService         The stream reader {@link ExecutorService} for normal priority uploads
      * @param priorityExecutorService The stream read {@link ExecutorService} for high priority uploads
      */
-    public AsyncTransferManager(long minimumPartSize, ExecutorService executorService, ExecutorService priorityExecutorService) {
+    public AsyncTransferManager(
+        long minimumPartSize,
+        ExecutorService executorService,
+        ExecutorService priorityExecutorService,
+        ExecutorService urgentExecutorService
+    ) {
         this.executorService = executorService;
         this.priorityExecutorService = priorityExecutorService;
         this.minimumPartSize = minimumPartSize;
+        this.urgentExecutorService = urgentExecutorService;
     }
 
     /**
@@ -162,6 +169,7 @@ private void doUploadInParts(
                 s3AsyncClient,
                 executorService,
                 priorityExecutorService,
+                urgentExecutorService,
                 uploadRequest,
                 streamContext,
                 uploadId,
@@ -308,9 +316,14 @@ private void uploadInOneChunk(
             putObjectRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
             putObjectRequestBuilder.checksumCRC32(base64StringFromLong(uploadRequest.getExpectedChecksum()));
         }
-        ExecutorService streamReadExecutor = uploadRequest.getWritePriority() == WritePriority.HIGH
-            ? priorityExecutorService
-            : executorService;
+        ExecutorService streamReadExecutor;
+        if (uploadRequest.getWritePriority() == WritePriority.URGENT) {
+            streamReadExecutor = urgentExecutorService;
+        } else if (uploadRequest.getWritePriority() == WritePriority.HIGH) {
+            streamReadExecutor = priorityExecutorService;
+        } else {
+            streamReadExecutor = executorService;
+        }
         // Buffered stream is needed to allow mark and reset ops during IO errors so that only buffered
         // data can be retried instead of retrying whole file by the application.
         InputStream inputStream = new BufferedInputStream(inputStreamContainer.getInputStream(), (int) (ByteSizeUnit.MB.toBytes(1) + 1));
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
index a4bfe11383b4f..8e1926d40302f 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
@@ -302,7 +302,7 @@ protected S3Repository createRepository(
             ClusterService clusterService,
             RecoverySettings recoverySettings
         ) {
-            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, false) {
+            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, null, false) {
                 @Override
                 protected void assertSnapshotOrGenericThread() {
                     // eliminate thread name check as we create repo manually on test/main threads
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java
index e9fe557ab751a..de9ad46bb222d 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java
@@ -44,12 +44,12 @@ public void testCachedClientsAreReleased() {
         final S3ClientSettings otherClientSettings = s3AsyncService.settings(metadata2);
         assertSame(clientSettings, otherClientSettings);
         final AmazonAsyncS3Reference reference = SocketAccess.doPrivileged(
-            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer)
+            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer)
         );
         reference.close();
         s3AsyncService.close();
         final AmazonAsyncS3Reference referenceReloaded = SocketAccess.doPrivileged(
-            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer)
+            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer)
         );
         assertNotSame(referenceReloaded, reference);
         referenceReloaded.close();
@@ -79,12 +79,12 @@ public void testCachedClientsWithCredentialsAreReleased() {
         final S3ClientSettings otherClientSettings = s3AsyncService.settings(metadata2);
         assertSame(clientSettings, otherClientSettings);
         final AmazonAsyncS3Reference reference = SocketAccess.doPrivileged(
-            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer)
+            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer)
         );
         reference.close();
         s3AsyncService.close();
         final AmazonAsyncS3Reference referenceReloaded = SocketAccess.doPrivileged(
-            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer)
+            () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer)
         );
         assertNotSame(referenceReloaded, reference);
         referenceReloaded.close();
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
index 6eb8faa746d34..7c67519f2f3b0 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
@@ -266,10 +266,11 @@ public void verifySingleChunkUploadCallCount(boolean finalizeUploadFailure) {
         @Override
         public AmazonAsyncS3Reference client(
             RepositoryMetadata repositoryMetadata,
+            AsyncExecutorContainer urgentExecutorBuilder,
             AsyncExecutorContainer priorityExecutorBuilder,
             AsyncExecutorContainer normalExecutorBuilder
         ) {
-            return new AmazonAsyncS3Reference(AmazonAsyncS3WithCredentials.create(asyncClient, asyncClient, null));
+            return new AmazonAsyncS3Reference(AmazonAsyncS3WithCredentials.create(asyncClient, asyncClient, asyncClient, null));
         }
     }
 
@@ -393,9 +394,11 @@ private S3BlobStore createBlobStore() {
             new AsyncTransferManager(
                 S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
                 asyncExecutorContainer.getStreamReader(),
+                asyncExecutorContainer.getStreamReader(),
                 asyncExecutorContainer.getStreamReader()
             ),
             asyncExecutorContainer,
+            asyncExecutorContainer,
             asyncExecutorContainer
         );
     }
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
index a2214f5218991..ceab06bd051e9 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -221,9 +221,11 @@ protected AsyncMultiStreamBlobContainer createBlobContainer(
                 new AsyncTransferManager(
                     S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
                     asyncExecutorContainer.getStreamReader(),
+                    asyncExecutorContainer.getStreamReader(),
                     asyncExecutorContainer.getStreamReader()
                 ),
                 asyncExecutorContainer,
+                asyncExecutorContainer,
                 asyncExecutorContainer
             )
         ) {
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
index 2701cae6a733b..58ad290a31e85 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
@@ -935,7 +935,7 @@ public void testReadBlobAsyncMultiPart() throws Exception {
 
         final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
         final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference(
-            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null)
+            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, s3AsyncClient, null)
         );
 
         final S3BlobStore blobStore = mock(S3BlobStore.class);
@@ -993,7 +993,7 @@ public void testReadBlobAsyncSinglePart() throws Exception {
 
         final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
         final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference(
-            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null)
+            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, s3AsyncClient, null)
         );
         final S3BlobStore blobStore = mock(S3BlobStore.class);
         final BlobPath blobPath = new BlobPath();
@@ -1048,7 +1048,7 @@ public void testReadBlobAsyncFailure() throws Exception {
 
         final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
         final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference(
-            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null)
+            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, s3AsyncClient, null)
         );
 
         final S3BlobStore blobStore = mock(S3BlobStore.class);
@@ -1091,7 +1091,7 @@ public void testReadBlobAsyncOnCompleteFailureMissingData() throws Exception {
 
         final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class);
         final AmazonAsyncS3Reference amazonAsyncS3Reference = new AmazonAsyncS3Reference(
-            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, null)
+            AmazonAsyncS3WithCredentials.create(s3AsyncClient, s3AsyncClient, s3AsyncClient, null)
         );
 
         final S3BlobStore blobStore = mock(S3BlobStore.class);
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
index e65ca69a5047b..6fec535ae6301 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
@@ -168,6 +168,7 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) {
             null,
             null,
             null,
+            null,
             false
         ) {
             @Override
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
index 97a746cdeed93..2437547a80a6f 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
@@ -64,6 +64,7 @@ public void setUp() throws Exception {
         asyncTransferManager = new AsyncTransferManager(
             ByteSizeUnit.MB.toBytes(5),
             Executors.newSingleThreadExecutor(),
+            Executors.newSingleThreadExecutor(),
             Executors.newSingleThreadExecutor()
         );
         super.setUp();
diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
index b8c0b52f93a3c..3f341c878c3c7 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
@@ -15,5 +15,6 @@
  */
 public enum WritePriority {
     NORMAL,
-    HIGH
+    HIGH,
+    URGENT
 }
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index ae4a3fab9852d..025ad075d83b6 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -378,7 +378,7 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException
             result.set(globalMetadataContainer.path().buildAsString() + globalMetadataFilename);
         }, ex -> { throw new GlobalMetadataTransferException(ex.getMessage(), ex); }), latch);
 
-        GLOBAL_METADATA_FORMAT.writeAsync(
+        GLOBAL_METADATA_FORMAT.writeAsyncWithUrgentPriority(
             clusterState.metadata(),
             globalMetadataContainer,
             globalMetadataFilename,
@@ -510,7 +510,7 @@ private void writeIndexMetadataAsync(
             ex -> latchedActionListener.onFailure(new IndexMetadataTransferException(indexMetadata.getIndex().toString(), ex))
         );
 
-        INDEX_METADATA_FORMAT.writeAsync(
+        INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority(
             indexMetadata,
             indexMetadataContainer,
             indexMetadataFilename,
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index e280141c12bc1..3e6052a5ef820 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -197,21 +197,56 @@ public void write(
     }
 
     /**
-     * Writes blob with resolving the blob name using {@link #blobName} method.
-     * Leverages the multipart upload if supported by the blobContainer.
+     * Internally calls {@link #writeAsyncWithPriority} with {@link WritePriority#NORMAL}
+     */
+    public void writeAsync(
+        final T obj,
+        final BlobContainer blobContainer,
+        final String name,
+        final Compressor compressor,
+        ActionListener listener,
+        final ToXContent.Params params
+    ) throws IOException {
+        // use NORMAL priority by default
+        this.writeAsyncWithPriority(obj, blobContainer, name, compressor, WritePriority.NORMAL, listener, params);
+    }
+
+    /**
+     * Internally calls {@link #writeAsyncWithPriority} with {@link WritePriority#URGENT}
+     * 
                              
+     * NOTE: We use this method to upload urgent priority objects like cluster state to remote stores.
+     * Use {@link #writeAsync(ToXContent, BlobContainer, String, Compressor, ActionListener, ToXContent.Params)} for
+     * other use cases.
+     */
+    public void writeAsyncWithUrgentPriority(
+        final T obj,
+        final BlobContainer blobContainer,
+        final String name,
+        final Compressor compressor,
+        ActionListener listener,
+        final ToXContent.Params params
+    ) throws IOException {
+        this.writeAsyncWithPriority(obj, blobContainer, name, compressor, WritePriority.URGENT, listener, params);
+    }
+
+    /**
+     * Method to writes blob with resolving the blob name using {@link #blobName} method with specified
+     * {@link WritePriority}. Leverages the multipart upload if supported by the blobContainer.
      *
      * @param obj                 object to be serialized
      * @param blobContainer       blob container
      * @param name                blob name
      * @param compressor          whether to use compression
+     * @param priority            write priority to be used
      * @param listener            listener to listen to write result
      * @param params              ToXContent params
      */
-    public void writeAsync(
+    private void writeAsyncWithPriority(
         final T obj,
         final BlobContainer blobContainer,
         final String name,
         final Compressor compressor,
+        final WritePriority priority,
         ActionListener listener,
         final ToXContent.Params params
     ) throws IOException {
@@ -222,7 +257,7 @@ public void writeAsync(
         }
         final String blobName = blobName(name);
         final BytesReference bytes = serialize(obj, blobName, compressor, params);
-        final String resourceDescription = "ChecksumBlobStoreFormat.writeAsync(blob=\"" + blobName + "\")";
+        final String resourceDescription = "ChecksumBlobStoreFormat.writeAsyncWithPriority(blob=\"" + blobName + "\")";
         try (IndexInput input = new ByteArrayIndexInput(resourceDescription, BytesReference.toBytes(bytes))) {
             long expectedChecksum;
             try {
@@ -242,7 +277,7 @@ public void writeAsync(
                     blobName,
                     bytes.length(),
                     true,
-                    WritePriority.HIGH,
+                    priority,
                     (size, position) -> new OffsetRangeIndexInputStream(input, size, position),
                     expectedChecksum,
                     ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported()
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 4be5fc03c2a6d..173e15b8eca37 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -273,7 +273,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
             new BytesArray(writtenBytes)
         );
 
-        assertEquals(capturedWriteContext.getWritePriority(), WritePriority.HIGH);
+        assertEquals(capturedWriteContext.getWritePriority(), WritePriority.URGENT);
         assertEquals(writtenIndexMetadata.getNumberOfShards(), 1);
         assertEquals(writtenIndexMetadata.getNumberOfReplicas(), 0);
         assertEquals(writtenIndexMetadata.getIndex().getName(), "test-index");
diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
index c114b56bd0b39..c5f36fcc01983 100644
--- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java
@@ -43,6 +43,7 @@
 import org.opensearch.common.blobstore.fs.FsBlobStore;
 import org.opensearch.common.blobstore.stream.read.ReadContext;
 import org.opensearch.common.blobstore.stream.write.WriteContext;
+import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.common.compress.DeflateCompressor;
 import org.opensearch.common.io.Streams;
 import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -65,8 +66,13 @@
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
+import org.mockito.ArgumentCaptor;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 public class BlobStoreFormatTests extends OpenSearchTestCase {
 
@@ -128,44 +134,36 @@ public void testBlobStoreAsyncOperations() throws IOException, InterruptedExcept
             BlobPath.cleanPath(),
             null
         );
+        MockFsVerifyingBlobContainer spyContainer = spy(mockBlobContainer);
         ChecksumBlobStoreFormat checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent);
-
+        ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
+        ArgumentCaptor writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class);
         CountDownLatch latch = new CountDownLatch(2);
 
-        ActionListener actionListener = new ActionListener<>() {
-            @Override
-            public void onResponse(Void unused) {
-                logger.info("---> Async write succeeded");
-                latch.countDown();
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                logger.info("---> Failure in async write");
-                throw new RuntimeException("async write should not fail");
-            }
-        };
-
         // Write blobs in different formats
         checksumSMILE.writeAsync(
             new BlobObj("checksum smile"),
-            mockBlobContainer,
+            spyContainer,
             "check-smile",
             CompressorRegistry.none(),
-            actionListener,
+            getVoidActionListener(latch),
             ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS
         );
         checksumSMILE.writeAsync(
             new BlobObj("checksum smile compressed"),
-            mockBlobContainer,
+            spyContainer,
             "check-smile-comp",
             CompressorRegistry.getCompressor(DeflateCompressor.NAME),
-            actionListener,
+            getVoidActionListener(latch),
             ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS
         );
 
         latch.await();
 
+        verify(spyContainer, times(2)).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture());
+        assertEquals(2, writeContextArgumentCaptor.getAllValues().size());
+        writeContextArgumentCaptor.getAllValues()
+            .forEach(writeContext -> assertEquals(WritePriority.NORMAL, writeContext.getWritePriority()));
         // Assert that all checksum blobs can be read
         assertEquals(checksumSMILE.read(mockBlobContainer.getDelegate(), "check-smile", xContentRegistry()).getText(), "checksum smile");
         assertEquals(
@@ -174,6 +172,39 @@ public void onFailure(Exception e) {
         );
     }
 
+    public void testBlobStorePriorityAsyncOperation() throws IOException, InterruptedException {
+        BlobStore blobStore = createTestBlobStore();
+        MockFsVerifyingBlobContainer mockBlobContainer = new MockFsVerifyingBlobContainer(
+            (FsBlobStore) blobStore,
+            BlobPath.cleanPath(),
+            null
+        );
+        MockFsVerifyingBlobContainer spyContainer = spy(mockBlobContainer);
+        ChecksumBlobStoreFormat checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent);
+
+        ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
+        ArgumentCaptor writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class);
+        CountDownLatch latch = new CountDownLatch(1);
+
+        // Write blobs in different formats
+        checksumSMILE.writeAsyncWithUrgentPriority(
+            new BlobObj("cluster state diff"),
+            spyContainer,
+            "cluster-state-diff",
+            CompressorRegistry.none(),
+            getVoidActionListener(latch),
+            ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS
+        );
+        latch.await();
+
+        verify(spyContainer).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture());
+        assertEquals(WritePriority.URGENT, writeContextArgumentCaptor.getValue().getWritePriority());
+        assertEquals(
+            checksumSMILE.read(mockBlobContainer.getDelegate(), "cluster-state-diff", xContentRegistry()).getText(),
+            "cluster state diff"
+        );
+    }
+
     public void testBlobStoreOperations() throws IOException {
         BlobStore blobStore = createTestBlobStore();
         BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
@@ -228,6 +259,24 @@ public void testBlobCorruption() throws IOException {
         }
     }
 
+    private ActionListener getVoidActionListener(CountDownLatch latch) {
+        ActionListener actionListener = new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {
+                logger.info("---> Async write succeeded");
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.info("---> Failure in async write");
+                throw new RuntimeException("async write should not fail");
+            }
+        };
+
+        return actionListener;
+    }
+
     protected BlobStore createTestBlobStore() throws IOException {
         return new FsBlobStore(randomIntBetween(1, 8) * 1024, createTempDir(), false);
     }

From 54e74a84437238c6154b0a15d209a9a1ecbaa4bb Mon Sep 17 00:00:00 2001
From: Aman Khare <85096200+amkhar@users.noreply.github.com>
Date: Tue, 24 Oct 2023 14:28:29 +0530
Subject: [PATCH 227/388] Add cluster state stats (#10670)

* Add cluster state update stats along with remote upload stats around success/ failure, latency metric

Signed-off-by: Aman Khare 
---
 CHANGELOG.md                                  |   1 +
 .../discovery/ClusterManagerDisruptionIT.java |   3 +
 .../remote/RemoteClusterStateServiceIT.java   |  43 ++++++
 .../coordination/CoordinationState.java       |   6 +
 .../cluster/coordination/Coordinator.java     |  12 +-
 .../coordination/InMemoryPersistedState.java  |   5 +
 .../coordination/PersistedStateStats.java     | 126 ++++++++++++++++++
 .../cluster/service/ClusterStateStats.java    | 120 +++++++++++++++++
 .../cluster/service/MasterService.java        |  23 +++-
 .../opensearch/discovery/DiscoveryStats.java  |  21 ++-
 .../opensearch/gateway/GatewayMetaState.java  |  13 ++
 .../remote/RemoteClusterStateService.java     |  20 ++-
 .../remote/RemotePersistenceStats.java        |  37 +++++
 .../cluster/node/stats/NodeStatsTests.java    |  29 +++-
 .../cluster/service/MasterServiceTests.java   |   3 +
 .../GatewayMetaStatePersistedStateTests.java  |  22 +++
 .../RemoteClusterStateServiceTests.java       |  34 +++++
 .../AbstractCoordinatorTestCase.java          |   5 +
 18 files changed, 511 insertions(+), 12 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
 create mode 100644 server/src/main/java/org/opensearch/cluster/service/ClusterStateStats.java
 create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b0d9720ad208..8c7e3ee151d64 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -96,6 +96,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
 - Added cluster setting cluster.restrict.index.replication_type to restrict setting of index setting replication type ([#10866](https://github.com/opensearch-project/OpenSearch/pull/10866))
+- Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java
index 1463c45aa9b2f..79f6ba6dfa642 100644
--- a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java
@@ -39,6 +39,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.NoClusterManagerBlockService;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.service.ClusterStateStats;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
@@ -199,6 +200,8 @@ public void testIsolateClusterManagerAndVerifyClusterStateConsensus() throws Exc
                 }
 
             }
+            ClusterStateStats clusterStateStats = internalCluster().clusterService().getClusterManagerService().getClusterStateStats();
+            assertTrue(clusterStateStats.getUpdateFailed() > 0);
         });
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
index 7304304e522f8..59eef3c06844b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
@@ -8,9 +8,12 @@
 
 package org.opensearch.gateway.remote;
 
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.discovery.DiscoveryStats;
 import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
@@ -19,6 +22,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
@@ -94,6 +98,45 @@ public void testFullClusterRestoreStaleDelete() throws Exception {
         assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards());
     }
 
+    public void testRemoteStateStats() {
+        int shardCount = randomIntBetween(1, 2);
+        int replicaCount = 1;
+        int dataNodeCount = shardCount * (replicaCount + 1);
+        int clusterManagerNodeCount = 1;
+        prepareCluster(clusterManagerNodeCount, dataNodeCount, INDEX_NAME, replicaCount, shardCount);
+        String clusterManagerNode = internalCluster().getClusterManagerName();
+        String dataNode = internalCluster().getDataNodeNames().stream().collect(Collectors.toList()).get(0);
+
+        // Fetch _nodes/stats
+        NodesStatsResponse nodesStatsResponse = client().admin()
+            .cluster()
+            .prepareNodesStats(clusterManagerNode)
+            .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
+            .get();
+
+        // assert cluster state stats
+        DiscoveryStats discoveryStats = nodesStatsResponse.getNodes().get(0).getDiscoveryStats();
+
+        assertNotNull(discoveryStats.getClusterStateStats());
+        assertTrue(discoveryStats.getClusterStateStats().getUpdateSuccess() > 1);
+        assertEquals(0, discoveryStats.getClusterStateStats().getUpdateFailed());
+        assertTrue(discoveryStats.getClusterStateStats().getUpdateTotalTimeInMillis() > 0);
+        // assert remote state stats
+        assertTrue(discoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 1);
+        assertEquals(0, discoveryStats.getClusterStateStats().getPersistenceStats().get(0).getFailedCount());
+        assertTrue(discoveryStats.getClusterStateStats().getPersistenceStats().get(0).getTotalTimeInMillis() > 0);
+
+        NodesStatsResponse nodesStatsResponseDataNode = client().admin()
+            .cluster()
+            .prepareNodesStats(dataNode)
+            .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
+            .get();
+        // assert cluster state stats for data node
+        DiscoveryStats dataNodeDiscoveryStats = nodesStatsResponseDataNode.getNodes().get(0).getDiscoveryStats();
+        assertNotNull(dataNodeDiscoveryStats.getClusterStateStats());
+        assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getUpdateSuccess());
+    }
+
     private void setReplicaCount(int replicaCount) {
         client().admin()
             .indices()
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java
index a339852e6ed8d..987a3e3ffa7d3 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java
@@ -638,6 +638,12 @@ public interface PersistedState extends Closeable {
          */
         void setLastAcceptedState(ClusterState clusterState);
 
+        /**
+         * Returns the stats for the persistence layer for {@link CoordinationState}.
+         * @return PersistedStateStats
+         */
+        PersistedStateStats getStats();
+
         /**
          * Marks the last accepted cluster state as committed.
          * After a successful call to this method, {@link #getLastAcceptedState()} should return the last cluster state that was set,
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
index eb30460ca1b7f..a4ffab7fb70c9 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
@@ -56,6 +56,7 @@
 import org.opensearch.cluster.service.ClusterApplier;
 import org.opensearch.cluster.service.ClusterApplier.ClusterApplyListener;
 import org.opensearch.cluster.service.ClusterManagerService;
+import org.opensearch.cluster.service.ClusterStateStats;
 import org.opensearch.common.Booleans;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.Priority;
@@ -865,7 +866,16 @@ protected void doStart() {
 
     @Override
     public DiscoveryStats stats() {
-        return new DiscoveryStats(new PendingClusterStateStats(0, 0, 0), publicationHandler.stats());
+        ClusterStateStats clusterStateStats = clusterManagerService.getClusterStateStats();
+        ArrayList stats = new ArrayList<>();
+        Stream.of(PersistedStateRegistry.PersistedStateType.values()).forEach(stateType -> {
+            if (persistedStateRegistry.getPersistedState(stateType) != null
+                && persistedStateRegistry.getPersistedState(stateType).getStats() != null) {
+                stats.add(persistedStateRegistry.getPersistedState(stateType).getStats());
+            }
+        });
+        clusterStateStats.setPersistenceStats(stats);
+        return new DiscoveryStats(new PendingClusterStateStats(0, 0, 0), publicationHandler.stats(), clusterStateStats);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/InMemoryPersistedState.java b/server/src/main/java/org/opensearch/cluster/coordination/InMemoryPersistedState.java
index 67ef82ee7b2e9..b77ede5471534 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/InMemoryPersistedState.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/InMemoryPersistedState.java
@@ -65,6 +65,11 @@ public void setLastAcceptedState(ClusterState clusterState) {
         this.acceptedState = clusterState;
     }
 
+    @Override
+    public PersistedStateStats getStats() {
+        return null;
+    }
+
     @Override
     public long getCurrentTerm() {
         return currentTerm;
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java b/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
new file mode 100644
index 0000000000000..1dc20e564ade2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
@@ -0,0 +1,126 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.coordination;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Persisted cluster state related stats.
+ *
+ * @opensearch.internal
+ */
+public class PersistedStateStats implements Writeable, ToXContentObject {
+    private String statsName;
+    private AtomicLong totalTimeInMillis = new AtomicLong(0);
+    private AtomicLong failedCount = new AtomicLong(0);
+    private AtomicLong successCount = new AtomicLong(0);
+    private Map extendedFields = new HashMap<>(); // keeping minimal extensibility
+
+    public PersistedStateStats(String statsName) {
+        this.statsName = statsName;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(successCount.get());
+        out.writeVLong(failedCount.get());
+        out.writeVLong(totalTimeInMillis.get());
+        if (extendedFields.size() > 0) {
+            out.writeBoolean(true);
+            out.writeVInt(extendedFields.size());
+            for (Map.Entry extendedField : extendedFields.entrySet()) {
+                out.writeString(extendedField.getKey());
+                out.writeVLong(extendedField.getValue().get());
+            }
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+
+    public PersistedStateStats(StreamInput in) throws IOException {
+        this.successCount = new AtomicLong(in.readVLong());
+        this.failedCount = new AtomicLong(in.readVLong());
+        this.totalTimeInMillis = new AtomicLong(in.readVLong());
+        if (in.readBoolean()) {
+            int extendedFieldsSize = in.readVInt();
+            this.extendedFields = new HashMap<>();
+            for (int fieldNumber = 0; fieldNumber < extendedFieldsSize; fieldNumber++) {
+                extendedFields.put(in.readString(), new AtomicLong(in.readVLong()));
+            }
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(statsName);
+        builder.field(Fields.SUCCESS_COUNT, getSuccessCount());
+        builder.field(Fields.FAILED_COUNT, getFailedCount());
+        builder.field(Fields.TOTAL_TIME_IN_MILLIS, getTotalTimeInMillis());
+        if (extendedFields.size() > 0) {
+            for (Map.Entry extendedField : extendedFields.entrySet()) {
+                builder.field(extendedField.getKey(), extendedField.getValue().get());
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public void stateFailed() {
+        failedCount.incrementAndGet();
+    }
+
+    public void stateSucceeded() {
+        successCount.incrementAndGet();
+    }
+
+    /**
+     * Expects user to send time taken in milliseconds.
+     *
+     * @param timeTakenInUpload time taken in uploading the cluster state to remote
+     */
+    public void stateTook(long timeTakenInUpload) {
+        totalTimeInMillis.addAndGet(timeTakenInUpload);
+    }
+
+    public long getTotalTimeInMillis() {
+        return totalTimeInMillis.get();
+    }
+
+    public long getFailedCount() {
+        return failedCount.get();
+    }
+
+    public long getSuccessCount() {
+        return successCount.get();
+    }
+
+    protected void addToExtendedFields(String extendedField, AtomicLong extendedFieldValue) {
+        this.extendedFields.put(extendedField, extendedFieldValue);
+    }
+
+    /**
+     * Fields for parsing and toXContent
+     *
+     * @opensearch.internal
+     */
+    static final class Fields {
+        static final String SUCCESS_COUNT = "success_count";
+        static final String TOTAL_TIME_IN_MILLIS = "total_time_in_millis";
+        static final String FAILED_COUNT = "failed_count";
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterStateStats.java b/server/src/main/java/org/opensearch/cluster/service/ClusterStateStats.java
new file mode 100644
index 0000000000000..96683ce720d0b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterStateStats.java
@@ -0,0 +1,120 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.service;
+
+import org.opensearch.cluster.coordination.PersistedStateStats;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Cluster state related stats.
+ *
+ * @opensearch.internal
+ */
+public class ClusterStateStats implements Writeable, ToXContentObject {
+
+    private AtomicLong updateSuccess = new AtomicLong(0);
+    private AtomicLong updateTotalTimeInMillis = new AtomicLong(0);
+    private AtomicLong updateFailed = new AtomicLong(0);
+    private List persistenceStats = new ArrayList<>();
+
+    public ClusterStateStats() {}
+
+    public long getUpdateSuccess() {
+        return updateSuccess.get();
+    }
+
+    public long getUpdateTotalTimeInMillis() {
+        return updateTotalTimeInMillis.get();
+    }
+
+    public long getUpdateFailed() {
+        return updateFailed.get();
+    }
+
+    public List getPersistenceStats() {
+        return persistenceStats;
+    }
+
+    public void stateUpdated() {
+        updateSuccess.incrementAndGet();
+    }
+
+    public void stateUpdateFailed() {
+        updateFailed.incrementAndGet();
+    }
+
+    public void stateUpdateTook(long stateUpdateTime) {
+        updateTotalTimeInMillis.addAndGet(stateUpdateTime);
+    }
+
+    public ClusterStateStats setPersistenceStats(List persistenceStats) {
+        this.persistenceStats = persistenceStats;
+        return this;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(updateSuccess.get());
+        out.writeVLong(updateTotalTimeInMillis.get());
+        out.writeVLong(updateFailed.get());
+        out.writeVInt(persistenceStats.size());
+        for (PersistedStateStats stats : persistenceStats) {
+            stats.writeTo(out);
+        }
+    }
+
+    public ClusterStateStats(StreamInput in) throws IOException {
+        this.updateSuccess = new AtomicLong(in.readVLong());
+        this.updateTotalTimeInMillis = new AtomicLong(in.readVLong());
+        this.updateFailed = new AtomicLong(in.readVLong());
+        int persistedStatsSize = in.readVInt();
+        this.persistenceStats = new ArrayList<>();
+        for (int statsNumber = 0; statsNumber < persistedStatsSize; statsNumber++) {
+            PersistedStateStats stats = new PersistedStateStats(in);
+            this.persistenceStats.add(stats);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.CLUSTER_STATE_STATS);
+        builder.startObject(Fields.OVERALL);
+        builder.field(Fields.UPDATE_COUNT, getUpdateSuccess());
+        builder.field(Fields.TOTAL_TIME_IN_MILLIS, getUpdateTotalTimeInMillis());
+        builder.field(Fields.FAILED_COUNT, getUpdateFailed());
+        builder.endObject();
+        for (PersistedStateStats stats : persistenceStats) {
+            stats.toXContent(builder, params);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Fields for parsing and toXContent
+     *
+     * @opensearch.internal
+     */
+    static final class Fields {
+        static final String CLUSTER_STATE_STATS = "cluster_state_stats";
+        static final String OVERALL = "overall";
+        static final String UPDATE_COUNT = "update_count";
+        static final String TOTAL_TIME_IN_MILLIS = "total_time_in_millis";
+        static final String FAILED_COUNT = "failed_count";
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
index 563b69dfd0e2a..07c3f93ae6486 100644
--- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
@@ -112,7 +112,9 @@ public class MasterService extends AbstractLifecycleComponent {
 
     static final String CLUSTER_MANAGER_UPDATE_THREAD_NAME = "clusterManagerService#updateTask";
 
-    /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #CLUSTER_MANAGER_UPDATE_THREAD_NAME} */
+    /**
+     * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #CLUSTER_MANAGER_UPDATE_THREAD_NAME}
+     */
     @Deprecated
     static final String MASTER_UPDATE_THREAD_NAME = "masterService#updateTask";
 
@@ -130,6 +132,7 @@ public class MasterService extends AbstractLifecycleComponent {
     private volatile Batcher taskBatcher;
     protected final ClusterManagerTaskThrottler clusterManagerTaskThrottler;
     private final ClusterManagerThrottlingStats throttlingStats;
+    private final ClusterStateStats stateStats;
 
     public MasterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
         this.nodeName = Objects.requireNonNull(Node.NODE_NAME_SETTING.get(settings));
@@ -147,6 +150,7 @@ public MasterService(Settings settings, ClusterSettings clusterSettings, ThreadP
             this::getMinNodeVersion,
             throttlingStats
         );
+        this.stateStats = new ClusterStateStats();
         this.threadPool = threadPool;
     }
 
@@ -339,7 +343,7 @@ private TimeValue getTimeSince(long startTimeNanos) {
         return TimeValue.timeValueMillis(TimeValue.nsecToMSec(threadPool.preciseRelativeTimeInNanos() - startTimeNanos));
     }
 
-    protected void publish(ClusterChangedEvent clusterChangedEvent, TaskOutputs taskOutputs, long startTimeMillis) {
+    protected void publish(ClusterChangedEvent clusterChangedEvent, TaskOutputs taskOutputs, long startTimeNanos) {
         final PlainActionFuture fut = new PlainActionFuture() {
             @Override
             protected boolean blockingAllowed() {
@@ -352,8 +356,12 @@ protected boolean blockingAllowed() {
         try {
             FutureUtils.get(fut);
             onPublicationSuccess(clusterChangedEvent, taskOutputs);
+            final long durationMillis = getTimeSince(startTimeNanos).millis();
+            stateStats.stateUpdateTook(durationMillis);
+            stateStats.stateUpdated();
         } catch (Exception e) {
-            onPublicationFailed(clusterChangedEvent, taskOutputs, startTimeMillis, e);
+            stateStats.stateUpdateFailed();
+            onPublicationFailed(clusterChangedEvent, taskOutputs, startTimeNanos, e);
         }
     }
 
@@ -464,7 +472,6 @@ public Builder incrementVersion(ClusterState clusterState) {
      * @param source     the source of the cluster state update task
      * @param updateTask the full context for the cluster state update
      *                   task
-     *
      */
     public  & ClusterStateTaskListener> void submitStateUpdateTask(
         String source,
@@ -490,7 +497,6 @@ public  & Cluster
      * @param listener callback after the cluster state update task
      *                 completes
      * @param       the type of the cluster state update task state
-     *
      */
     public  void submitStateUpdateTask(
         String source,
@@ -947,7 +953,7 @@ void onNoLongerClusterManager() {
     /**
      * Functionality for register task key to cluster manager node.
      *
-     * @param taskKey - task key of task
+     * @param taskKey           - task key of task
      * @param throttlingEnabled - throttling is enabled for task or not i.e does data node perform retries on it or not
      * @return throttling task key which needs to be passed while submitting task to cluster manager
      */
@@ -966,7 +972,6 @@ public ClusterManagerTaskThrottler.ThrottlingKey registerClusterManagerTask(Stri
      *                 that share the same executor will be executed
      *                 batches on this executor
      * @param       the type of the cluster state update task state
-     *
      */
     public  void submitStateUpdateTasks(
         final String source,
@@ -996,4 +1001,8 @@ public  void submitStateUpdateTasks(
         }
     }
 
+    public ClusterStateStats getClusterStateStats() {
+        return stateStats;
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java b/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java
index 665ecf77d7aa7..ea93ccd09ed39 100644
--- a/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java
+++ b/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java
@@ -32,8 +32,10 @@
 
 package org.opensearch.discovery;
 
+import org.opensearch.Version;
 import org.opensearch.cluster.coordination.PendingClusterStateStats;
 import org.opensearch.cluster.coordination.PublishClusterStateStats;
+import org.opensearch.cluster.service.ClusterStateStats;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -51,21 +53,31 @@ public class DiscoveryStats implements Writeable, ToXContentFragment {
 
     private final PendingClusterStateStats queueStats;
     private final PublishClusterStateStats publishStats;
+    private final ClusterStateStats clusterStateStats;
 
-    public DiscoveryStats(PendingClusterStateStats queueStats, PublishClusterStateStats publishStats) {
+    public DiscoveryStats(PendingClusterStateStats queueStats, PublishClusterStateStats publishStats, ClusterStateStats clusterStateStats) {
         this.queueStats = queueStats;
         this.publishStats = publishStats;
+        this.clusterStateStats = clusterStateStats;
     }
 
     public DiscoveryStats(StreamInput in) throws IOException {
         queueStats = in.readOptionalWriteable(PendingClusterStateStats::new);
         publishStats = in.readOptionalWriteable(PublishClusterStateStats::new);
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            clusterStateStats = in.readOptionalWriteable(ClusterStateStats::new);
+        } else {
+            clusterStateStats = null;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(queueStats);
         out.writeOptionalWriteable(publishStats);
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalWriteable(clusterStateStats);
+        }
     }
 
     @Override
@@ -77,6 +89,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (publishStats != null) {
             publishStats.toXContent(builder, params);
         }
+        if (clusterStateStats != null) {
+            clusterStateStats.toXContent(builder, params);
+        }
         builder.endObject();
         return builder;
     }
@@ -92,4 +107,8 @@ public PendingClusterStateStats getQueueStats() {
     public PublishClusterStateStats getPublishStats() {
         return publishStats;
     }
+
+    public ClusterStateStats getClusterStateStats() {
+        return clusterStateStats;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
index f855449c708d2..350a361a49a62 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -47,6 +47,7 @@
 import org.opensearch.cluster.coordination.InMemoryPersistedState;
 import org.opensearch.cluster.coordination.PersistedStateRegistry;
 import org.opensearch.cluster.coordination.PersistedStateRegistry.PersistedStateType;
+import org.opensearch.cluster.coordination.PersistedStateStats;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexTemplateMetadata;
 import org.opensearch.cluster.metadata.Manifest;
@@ -615,6 +616,12 @@ public void setLastAcceptedState(ClusterState clusterState) {
             lastAcceptedState = clusterState;
         }
 
+        @Override
+        public PersistedStateStats getStats() {
+            // Note: These stats are not published yet, will come in future
+            return null;
+        }
+
         private PersistedClusterStateService.Writer getWriterSafe() {
             final PersistedClusterStateService.Writer writer = persistenceWriter.get();
             if (writer == null) {
@@ -717,10 +724,16 @@ assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) ==
                 lastAcceptedManifest = manifest;
                 lastAcceptedState = clusterState;
             } catch (Exception e) {
+                remoteClusterStateService.writeMetadataFailed();
                 handleExceptionOnWrite(e);
             }
         }
 
+        @Override
+        public PersistedStateStats getStats() {
+            return remoteClusterStateService.getStats();
+        }
+
         private boolean verifyManifestAndClusterState(ClusterMetadataManifest manifest, ClusterState clusterState) {
             assert manifest != null : "ClusterMetadataManifest is null";
             assert clusterState != null : "ClusterState is null";
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 025ad075d83b6..329ebd0dcd2b8 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -159,7 +159,7 @@ public class RemoteClusterStateService implements Closeable {
     private volatile TimeValue globalMetadataUploadTimeout;
 
     private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false);
-
+    private final RemotePersistenceStats remoteStateStats;
     public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1;
     public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1;
     public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1;
@@ -193,6 +193,7 @@ public RemoteClusterStateService(
         clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold);
         clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout);
         clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout);
+        this.remoteStateStats = new RemotePersistenceStats();
     }
 
     private BlobStoreTransferService getBlobStoreTransferService() {
@@ -233,6 +234,8 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri
             false
         );
         final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos);
+        remoteStateStats.stateSucceeded();
+        remoteStateStats.stateTook(durationMillis);
         if (durationMillis >= slowWriteLoggingThreshold.getMillis()) {
             logger.warn(
                 "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " + "wrote full state with [{}] indices",
@@ -334,6 +337,8 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS);
 
         final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos);
+        remoteStateStats.stateSucceeded();
+        remoteStateStats.stateTook(durationMillis);
         if (durationMillis >= slowWriteLoggingThreshold.getMillis()) {
             logger.warn(
                 "writing cluster state took [{}ms] which is above the warn threshold of [{}]; "
@@ -1059,6 +1064,10 @@ public static String encodeString(String content) {
         return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8));
     }
 
+    public void writeMetadataFailed() {
+        getStats().stateFailed();
+    }
+
     /**
      * Exception for IndexMetadata transfer failures to remote
      */
@@ -1093,7 +1102,7 @@ public GlobalMetadataTransferException(String errorDesc, Throwable cause) {
      * @param clusterName name of the cluster
      * @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged
      */
-    private void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUIDs) {
+    void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUIDs) {
         clusterUUIDs.forEach(clusterUUID -> {
             getBlobStoreTransferService().deleteAsync(
                 ThreadPool.Names.REMOTE_PURGE,
@@ -1113,6 +1122,7 @@ public void onFailure(Exception e) {
                             ),
                             e
                         );
+                        remoteStateStats.cleanUpAttemptFailed();
                     }
                 }
             );
@@ -1228,8 +1238,10 @@ private void deleteClusterMetadata(
             logger.error("Error while fetching Remote Cluster Metadata manifests", e);
         } catch (IOException e) {
             logger.error("Error while deleting stale Remote Cluster Metadata files", e);
+            remoteStateStats.cleanUpAttemptFailed();
         } catch (Exception e) {
             logger.error("Unexpected error while deleting stale Remote Cluster Metadata files", e);
+            remoteStateStats.cleanUpAttemptFailed();
         }
     }
 
@@ -1260,4 +1272,8 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa
             deleteStaleUUIDsClusterMetadata(clusterName, new ArrayList<>(allClustersUUIDsInRemote));
         });
     }
+
+    public RemotePersistenceStats getStats() {
+        return remoteStateStats;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java
new file mode 100644
index 0000000000000..f2330846fa23e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote;
+
+import org.opensearch.cluster.coordination.PersistedStateStats;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Remote state related extended stats.
+ *
+ * @opensearch.internal
+ */
+public class RemotePersistenceStats extends PersistedStateStats {
+    static final String CLEANUP_ATTEMPT_FAILED_COUNT = "cleanup_attempt_failed_count";
+    static final String REMOTE_UPLOAD = "remote_upload";
+    private AtomicLong cleanupAttemptFailedCount = new AtomicLong(0);
+
+    public RemotePersistenceStats() {
+        super(REMOTE_UPLOAD);
+        addToExtendedFields(CLEANUP_ATTEMPT_FAILED_COUNT, cleanupAttemptFailedCount);
+    }
+
+    public void cleanUpAttemptFailed() {
+        cleanupAttemptFailedCount.incrementAndGet();
+    }
+
+    public long getCleanupAttemptFailedCount() {
+        return cleanupAttemptFailedCount.get();
+    }
+}
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index ebdd012006fb2..3050d1674a95b 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -36,10 +36,12 @@
 import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
 import org.opensearch.action.search.SearchRequestStats;
 import org.opensearch.cluster.coordination.PendingClusterStateStats;
+import org.opensearch.cluster.coordination.PersistedStateStats;
 import org.opensearch.cluster.coordination.PublishClusterStateStats;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.routing.WeightedRoutingStats;
 import org.opensearch.cluster.service.ClusterManagerThrottlingStats;
+import org.opensearch.cluster.service.ClusterStateStats;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.metrics.OperationStats;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -47,6 +49,7 @@
 import org.opensearch.core.indices.breaker.AllCircuitBreakerStats;
 import org.opensearch.core.indices.breaker.CircuitBreakerStats;
 import org.opensearch.discovery.DiscoveryStats;
+import org.opensearch.gateway.remote.RemotePersistenceStats;
 import org.opensearch.http.HttpStats;
 import org.opensearch.index.ReplicationStats;
 import org.opensearch.index.SegmentReplicationRejectionStats;
@@ -72,6 +75,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -349,6 +353,25 @@ public void testSerialization() throws IOException {
                         assertEquals(queueStats.getTotal(), deserializedDiscoveryStats.getQueueStats().getTotal());
                         assertEquals(queueStats.getPending(), deserializedDiscoveryStats.getQueueStats().getPending());
                     }
+                    ClusterStateStats stateStats = discoveryStats.getClusterStateStats();
+                    if (stateStats == null) {
+                        assertNull(deserializedDiscoveryStats.getClusterStateStats());
+                    } else {
+                        assertEquals(stateStats.getUpdateFailed(), deserializedDiscoveryStats.getClusterStateStats().getUpdateFailed());
+                        assertEquals(stateStats.getUpdateSuccess(), deserializedDiscoveryStats.getClusterStateStats().getUpdateSuccess());
+                        assertEquals(
+                            stateStats.getUpdateTotalTimeInMillis(),
+                            deserializedDiscoveryStats.getClusterStateStats().getUpdateTotalTimeInMillis()
+                        );
+                        assertEquals(1, deserializedDiscoveryStats.getClusterStateStats().getPersistenceStats().size());
+                        PersistedStateStats deserializedRemoteStateStats = deserializedDiscoveryStats.getClusterStateStats()
+                            .getPersistenceStats()
+                            .get(0);
+                        PersistedStateStats remoteStateStats = stateStats.getPersistenceStats().get(0);
+                        assertEquals(remoteStateStats.getFailedCount(), deserializedRemoteStateStats.getFailedCount());
+                        assertEquals(remoteStateStats.getSuccessCount(), deserializedRemoteStateStats.getSuccessCount());
+                        assertEquals(remoteStateStats.getTotalTimeInMillis(), deserializedRemoteStateStats.getTotalTimeInMillis());
+                    }
                 }
                 IngestStats ingestStats = nodeStats.getIngestStats();
                 IngestStats deserializedIngestStats = deserializedNodeStats.getIngestStats();
@@ -725,12 +748,16 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) {
         ScriptStats scriptStats = frequently()
             ? new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())
             : null;
+        ClusterStateStats stateStats = new ClusterStateStats();
+        RemotePersistenceStats remoteStateStats = new RemotePersistenceStats();
+        stateStats.setPersistenceStats(Arrays.asList(remoteStateStats));
         DiscoveryStats discoveryStats = frequently()
             ? new DiscoveryStats(
                 randomBoolean() ? new PendingClusterStateStats(randomInt(), randomInt(), randomInt()) : null,
                 randomBoolean()
                     ? new PublishClusterStateStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())
-                    : null
+                    : null,
+                randomBoolean() ? stateStats : null
             )
             : null;
         IngestStats ingestStats = null;
diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
index 9cdbe04e0a0e4..4c0ca826f5dcc 100644
--- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
@@ -691,6 +691,9 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
                     submittedTasksPerThread.get(entry.getKey()).get()
                 );
             }
+            // verify stats values after state is published
+            assertEquals(1, clusterManagerService.getClusterStateStats().getUpdateSuccess());
+            assertEquals(0, clusterManagerService.getClusterStateStats().getUpdateFailed());
         }
     }
 
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
index 1d5c2a0f01b5c..fd113ed4313d7 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
@@ -68,6 +68,7 @@
 import org.opensearch.gateway.PersistedClusterStateService.Writer;
 import org.opensearch.gateway.remote.ClusterMetadataManifest;
 import org.opensearch.gateway.remote.RemoteClusterStateService;
+import org.opensearch.gateway.remote.RemotePersistenceStats;
 import org.opensearch.index.recovery.RemoteStoreRestoreService;
 import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult;
 import org.opensearch.node.Node;
@@ -104,6 +105,7 @@
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.doCallRealMethod;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -779,6 +781,26 @@ public void testRemotePersistedStateExceptionOnFullStateUpload() throws IOExcept
         assertThrows(OpenSearchException.class, () -> remotePersistedState.setLastAcceptedState(clusterState));
     }
 
+    public void testRemotePersistedStateFailureStats() throws IOException {
+        RemotePersistenceStats remoteStateStats = new RemotePersistenceStats();
+        final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class);
+        final String previousClusterUUID = "prev-cluster-uuid";
+        Mockito.doThrow(IOException.class).when(remoteClusterStateService).writeFullMetadata(Mockito.any(), Mockito.any());
+        when(remoteClusterStateService.getStats()).thenReturn(remoteStateStats);
+        doCallRealMethod().when(remoteClusterStateService).writeMetadataFailed();
+        CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID);
+
+        final long clusterTerm = randomNonNegativeLong();
+        final ClusterState clusterState = createClusterState(
+            randomNonNegativeLong(),
+            Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build()
+        );
+
+        assertThrows(OpenSearchException.class, () -> remotePersistedState.setLastAcceptedState(clusterState));
+        assertEquals(1, remoteClusterStateService.getStats().getFailedCount());
+        assertEquals(0, remoteClusterStateService.getStats().getSuccessCount());
+    }
+
     public void testGatewayForRemoteState() throws IOException {
         MockGatewayMetaState gateway = null;
         try {
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 173e15b8eca37..5a43864f40c0c 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -324,6 +324,7 @@ public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOEx
             RemoteClusterStateService.IndexMetadataTransferException.class,
             () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10))
         );
+        assertEquals(0, remoteClusterStateService.getStats().getSuccessCount());
     }
 
     public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOException {
@@ -331,6 +332,7 @@ public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOExc
         remoteClusterStateService.start();
         final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata(clusterState, clusterState, null);
         Assert.assertThat(manifest, nullValue());
+        assertEquals(0, remoteClusterStateService.getStats().getSuccessCount());
     }
 
     public void testFailWriteIncrementalMetadataWhenTermChanged() {
@@ -991,6 +993,38 @@ public void testDeleteStaleClusterUUIDs() throws IOException {
         }
     }
 
+    public void testRemoteStateStats() throws IOException {
+        final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        mockBlobStoreObjects();
+        remoteClusterStateService.start();
+        final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid");
+
+        assertTrue(remoteClusterStateService.getStats() != null);
+        assertEquals(1, remoteClusterStateService.getStats().getSuccessCount());
+        assertEquals(0, remoteClusterStateService.getStats().getCleanupAttemptFailedCount());
+        assertEquals(0, remoteClusterStateService.getStats().getFailedCount());
+    }
+
+    public void testRemoteStateCleanupFailureStats() throws IOException {
+        BlobContainer blobContainer = mock(BlobContainer.class);
+        doThrow(IOException.class).when(blobContainer).delete();
+        when(blobStore.blobContainer(any())).thenReturn(blobContainer);
+        BlobPath blobPath = new BlobPath().add("random-path");
+        when((blobStoreRepository.basePath())).thenReturn(blobPath);
+        remoteClusterStateService.start();
+        remoteClusterStateService.deleteStaleUUIDsClusterMetadata("cluster1", Arrays.asList("cluster-uuid1"));
+        try {
+            assertBusy(() -> {
+                // wait for stats to get updated
+                assertTrue(remoteClusterStateService.getStats() != null);
+                assertEquals(0, remoteClusterStateService.getStats().getSuccessCount());
+                assertEquals(1, remoteClusterStateService.getStats().getCleanupAttemptFailedCount());
+            });
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public void testFileNames() {
         final Index index = new Index("test-index", "index-uuid");
         final Settings idxSettings = Settings.builder()
diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
index d24cc24d28579..28d7706fb1493 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -1016,6 +1016,11 @@ public void setLastAcceptedState(ClusterState clusterState) {
                 delegate.setLastAcceptedState(clusterState);
             }
 
+            @Override
+            public PersistedStateStats getStats() {
+                return null;
+            }
+
             @Override
             public void close() {
                 assertTrue(openPersistedStates.remove(this));

From 6f36752d9e84e95ce2280347cc26b0c9138b2d57 Mon Sep 17 00:00:00 2001
From: Sachin Kale 
Date: Tue, 24 Oct 2023 14:31:01 +0530
Subject: [PATCH 228/388] Sync translog to remote on primary activate (#10839)

---------

Signed-off-by: Sachin Kale 
Co-authored-by: Sachin Kale 
---
 .../remotestore/RemoteRestoreSnapshotIT.java  | 94 +++++++++++++++++++
 .../remotestore/RemoteStoreStatsIT.java       |  8 +-
 .../opensearch/index/shard/IndexShard.java    | 19 +++-
 .../index/shard/IndexShardTests.java          |  1 +
 4 files changed, 116 insertions(+), 6 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
index 865b2d13f189e..9e0b2a66467de 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
@@ -11,6 +11,7 @@
 import org.opensearch.action.DocWriteResponse;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.admin.indices.get.GetIndexRequest;
 import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.delete.DeleteResponse;
@@ -20,8 +21,13 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.core.index.Index;
 import org.opensearch.core.rest.RestStatus;
+import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.opensearch.snapshots.SnapshotInfo;
@@ -32,11 +38,15 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
@@ -345,6 +355,90 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
         assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 4);
     }
 
+    public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, ExecutionException, InterruptedException {
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNodes(2);
+
+        String indexName1 = "testindex1";
+        String snapshotRepoName = "test-restore-snapshot-repo";
+        String snapshotName1 = "test-restore-snapshot1";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+
+        createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
+
+        Settings indexSettings = getIndexSettings(1, 0).build();
+        createIndex(indexName1, indexSettings);
+
+        final int numDocsInIndex1 = randomIntBetween(20, 30);
+        indexDocuments(client(), indexName1, numDocsInIndex1);
+        flushAndRefresh(indexName1);
+        ensureGreen(indexName1);
+
+        logger.info("--> snapshot");
+        SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
+        assertThat(snapshotInfo1.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards()));
+        assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS));
+
+        assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get());
+        assertFalse(indexExists(indexName1));
+
+        RestoreSnapshotResponse restoreSnapshotResponse1 = client().admin()
+            .cluster()
+            .prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
+            .setWaitForCompletion(false)
+            .setIndices(indexName1)
+            .get();
+
+        assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED);
+        ensureGreen(indexName1);
+        assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
+
+        // Make sure remote translog is empty
+        String indexUUID = client().admin()
+            .indices()
+            .prepareGetSettings(indexName1)
+            .get()
+            .getSetting(indexName1, IndexMetadata.SETTING_INDEX_UUID);
+
+        Path remoteTranslogMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/metadata");
+        Path remoteTranslogDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/data");
+
+        try (
+            Stream translogMetadata = Files.list(remoteTranslogMetadataPath);
+            Stream translogData = Files.list(remoteTranslogDataPath)
+        ) {
+            assertTrue(translogData.count() > 0);
+            assertTrue(translogMetadata.count() > 0);
+        }
+
+        // Clear the local data before stopping the node. This will make sure that remote translog is empty.
+        IndexShard indexShard = getIndexShard(primaryNodeName(indexName1), indexName1);
+        try (Stream files = Files.list(indexShard.shardPath().resolveTranslog())) {
+            IOUtils.deleteFilesIgnoringExceptions(files.collect(Collectors.toList()));
+        }
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(indexName1)));
+
+        ensureRed(indexName1);
+
+        client().admin()
+            .cluster()
+            .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(false), PlainActionFuture.newFuture());
+
+        ensureGreen(indexName1);
+        assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
+    }
+
+    protected IndexShard getIndexShard(String node, String indexName) {
+        final Index index = resolveIndex(indexName);
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
+        IndexService indexService = indicesService.indexService(index);
+        assertNotNull(indexService);
+        final Optional shardId = indexService.shardIds().stream().findFirst();
+        return shardId.map(indexService::getShard).orElse(null);
+    }
+
     public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException {
         String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
         String primary = internalCluster().startDataOnlyNode();
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
index 5e91176ed0473..b1dbb0a900bc7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
@@ -581,21 +581,23 @@ public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exce
                 .getRemoteStoreStats();
             Arrays.stream(remoteStoreStats).forEach(statObject -> {
                 RemoteSegmentTransferTracker.Stats segmentStats = statObject.getSegmentStats();
+                RemoteTranslogTransferTracker.Stats translogStats = statObject.getTranslogStats();
                 if (statObject.getShardRouting().primary()) {
                     assertTrue(
                         segmentStats.totalUploadsSucceeded == 1
                             && segmentStats.totalUploadsStarted == segmentStats.totalUploadsSucceeded
                             && segmentStats.totalUploadsFailed == 0
                     );
+                    // On primary shard creation, we upload to remote translog post primary mode activation.
+                    // This changes upload stats to non-zero for primary shard.
+                    assertNonZeroTranslogUploadStatsNoFailures(translogStats);
                 } else {
                     assertTrue(
                         segmentStats.directoryFileTransferTrackerStats.transferredBytesStarted == 0
                             && segmentStats.directoryFileTransferTrackerStats.transferredBytesSucceeded == 0
                     );
+                    assertZeroTranslogUploadStats(translogStats);
                 }
-
-                RemoteTranslogTransferTracker.Stats translogStats = statObject.getTranslogStats();
-                assertZeroTranslogUploadStats(translogStats);
                 assertZeroTranslogDownloadStats(translogStats);
             });
         }, 5, TimeUnit.SECONDS);
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index fb4e9056153aa..3c348035ebbdd 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -640,7 +640,7 @@ public void updateShardState(
                     if (currentRouting.initializing() && currentRouting.isRelocationTarget() == false && newRouting.active()) {
                         // the cluster-manager started a recovering primary, activate primary mode.
                         replicationTracker.activatePrimaryMode(getLocalCheckpoint());
-                        ensurePeerRecoveryRetentionLeasesExist();
+                        postActivatePrimaryMode();
                     }
                 } else {
                     assert currentRouting.primary() == false : "term is only increased as part of primary promotion";
@@ -711,8 +711,7 @@ public void updateShardState(
                                 // are brought up to date.
                                 checkpointPublisher.publish(this, getLatestReplicationCheckpoint());
                             }
-
-                            ensurePeerRecoveryRetentionLeasesExist();
+                            postActivatePrimaryMode();
                             /*
                              * If this shard was serving as a replica shard when another shard was promoted to primary then
                              * its Lucene index was reset during the primary term transition. In particular, the Lucene index
@@ -3393,6 +3392,20 @@ assert getLocalCheckpoint() == primaryContext.getCheckpointStates().get(routingE
         synchronized (mutex) {
             replicationTracker.activateWithPrimaryContext(primaryContext); // make changes to primaryMode flag only under mutex
         }
+        postActivatePrimaryMode();
+    }
+
+    private void postActivatePrimaryMode() {
+        if (indexSettings.isRemoteStoreEnabled()) {
+            // We make sure to upload translog (even if it does not contain any operations) to remote translog.
+            // This helps to get a consistent state in remote store where both remote segment store and remote
+            // translog contains data.
+            try {
+                getEngine().translogManager().syncTranslog();
+            } catch (IOException e) {
+                logger.error("Failed to sync translog to remote from new primary", e);
+            }
+        }
         ensurePeerRecoveryRetentionLeasesExist();
     }
 
diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index 9ef9bec01cb38..fa3cf7676f55c 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -2745,6 +2745,7 @@ public void testRelocatedForRemoteTranslogBackedIndexWithAsyncDurability() throw
             AllocationId.newRelocation(routing.allocationId())
         );
         IndexShardTestCase.updateRoutingEntry(indexShard, routing);
+        indexDoc(indexShard, "_doc", "0");
         assertTrue(indexShard.isSyncNeeded());
         try {
             indexShard.relocated(routing.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {}, () -> {});

From 5bd413c588f48589c6fd6c4de4e87550271aecf8 Mon Sep 17 00:00:00 2001
From: Peter Nied 
Date: Tue, 24 Oct 2023 07:38:18 -0400
Subject: [PATCH 229/388] GHA to verify checklist items completion in PR
 descriptions (#10800)

Signed-off-by: Peter Nied 
---
 .github/workflows/pull-request-checks.yml | 28 +++++++++++++++++++++++
 CHANGELOG.md                              |  1 +
 2 files changed, 29 insertions(+)
 create mode 100644 .github/workflows/pull-request-checks.yml

diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
new file mode 100644
index 0000000000000..11998e36c2dbb
--- /dev/null
+++ b/.github/workflows/pull-request-checks.yml
@@ -0,0 +1,28 @@
+name: Pull Request Checks
+
+on:
+  pull_request:
+    types:
+      [
+        opened,
+        edited,
+        review_requested,
+        synchronize,
+        reopened,
+        ready_for_review,
+      ]
+
+jobs:
+  verify-description-checklist:
+    name: Verify Description Checklist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peternied/check-pull-request-description-checklist@v1
+        with:
+          checklist-items: |
+            New functionality includes testing.
+            All tests pass
+            New functionality has been documented.
+            New functionality has javadoc added
+            Commits are signed per the DCO using --signoff
+            Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c7e3ee151d64..b40878066960a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Download functionality of global metadata from remote store ([#10535](https://github.com/opensearch-project/OpenSearch/pull/10535))
 - [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
 - [AdmissionControl] Added changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting ([#9286](https://github.com/opensearch-project/OpenSearch/pull/9286))
+- GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0

From 91ac0846dbeb7379ae0772bf9f144cd628e9deac Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Wed, 25 Oct 2023 14:01:27 +0530
Subject: [PATCH 230/388] link previous cluster uuid to current cluster uuid
 even if current cluster uuid is not committed (#10832)

* link previous cluster uuid to current cluster uuid even if current cluster uuid is not committed

Signed-off-by: bansvaru 
---
 .../opensearch/gateway/GatewayMetaState.java  | 29 +++++++-------
 .../GatewayMetaStatePersistedStateTests.java  | 39 +++++++++++++++++++
 2 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
index 350a361a49a62..c3056276706a0 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -695,24 +695,21 @@ public void setLastAcceptedState(ClusterState clusterState) {
             try {
                 final ClusterMetadataManifest manifest;
                 if (shouldWriteFullClusterState(clusterState)) {
-                    if (clusterState.metadata().clusterUUIDCommitted() == true) {
-                        final Optional latestManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
-                            clusterState.getClusterName().value(),
+                    final Optional latestManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
+                        clusterState.getClusterName().value(),
+                        clusterState.metadata().clusterUUID()
+                    );
+                    if (latestManifest.isPresent()) {
+                        // The previous UUID should not change for the current UUID. So fetching the latest manifest
+                        // from remote store and getting the previous UUID.
+                        previousClusterUUID = latestManifest.get().getPreviousClusterUUID();
+                    } else {
+                        // When the user starts the cluster with remote state disabled but later enables the remote state,
+                        // there will not be any manifest for the current cluster UUID.
+                        logger.error(
+                            "Latest manifest is not present in remote store for cluster UUID: {}",
                             clusterState.metadata().clusterUUID()
                         );
-                        if (latestManifest.isPresent()) {
-                            // The previous UUID should not change for the current UUID. So fetching the latest manifest
-                            // from remote store and getting the previous UUID.
-                            previousClusterUUID = latestManifest.get().getPreviousClusterUUID();
-                        } else {
-                            // When the user starts the cluster with remote state disabled but later enables the remote state,
-                            // there will not be any manifest for the current cluster UUID.
-                            logger.error(
-                                "Latest manifest is not present in remote store for cluster UUID: {}",
-                                clusterState.metadata().clusterUUID()
-                            );
-                            previousClusterUUID = ClusterState.UNKNOWN_UUID;
-                        }
                     }
                     manifest = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID);
                 } else {
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
index fd113ed4313d7..74bae7b5eb7cf 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
@@ -87,10 +87,12 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
@@ -765,6 +767,43 @@ public void testRemotePersistedState() throws IOException {
         assertThat(remotePersistedState.getLastAcceptedState().metadata().clusterUUIDCommitted(), equalTo(true));
     }
 
+    public void testRemotePersistedStateNotCommitted() throws IOException {
+        final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class);
+        final String previousClusterUUID = "prev-cluster-uuid";
+        final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder()
+            .previousClusterUUID(previousClusterUUID)
+            .clusterTerm(1L)
+            .stateVersion(5L)
+            .build();
+        Mockito.when(remoteClusterStateService.getLatestClusterMetadataManifest(Mockito.any(), Mockito.any()))
+            .thenReturn(Optional.of(manifest));
+        Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())).thenReturn(manifest);
+
+        Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(manifest);
+        CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(
+            remoteClusterStateService,
+            ClusterState.UNKNOWN_UUID
+        );
+
+        assertThat(remotePersistedState.getLastAcceptedState(), nullValue());
+        assertThat(remotePersistedState.getCurrentTerm(), equalTo(0L));
+
+        final long clusterTerm = randomNonNegativeLong();
+        ClusterState clusterState = createClusterState(
+            randomNonNegativeLong(),
+            Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build()
+        );
+        clusterState = ClusterState.builder(clusterState)
+            .metadata(Metadata.builder(clusterState.getMetadata()).clusterUUID(randomAlphaOfLength(10)).clusterUUIDCommitted(false).build())
+            .build();
+
+        remotePersistedState.setLastAcceptedState(clusterState);
+        ArgumentCaptor previousClusterUUIDCaptor = ArgumentCaptor.forClass(String.class);
+        ArgumentCaptor clusterStateCaptor = ArgumentCaptor.forClass(ClusterState.class);
+        Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterStateCaptor.capture(), previousClusterUUIDCaptor.capture());
+        assertEquals(previousClusterUUID, previousClusterUUIDCaptor.getValue());
+    }
+
     public void testRemotePersistedStateExceptionOnFullStateUpload() throws IOException {
         final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class);
         final String previousClusterUUID = "prev-cluster-uuid";

From b5299f13e0ca9a5f6979e8cb50137682e777b095 Mon Sep 17 00:00:00 2001
From: Sachin Kale 
Date: Wed, 25 Oct 2023 14:46:44 +0530
Subject: [PATCH 231/388] Delete corrupted file to re-download from remote
 store (#10891)

---------

Signed-off-by: Sachin Kale 
Co-authored-by: Sachin Kale 
---
 .../opensearch/index/shard/IndexShard.java    |  5 +-
 .../index/shard/IndexShardTests.java          | 52 +++++++++++++++++++
 .../org/opensearch/test/CorruptionUtils.java  |  2 +-
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 3c348035ebbdd..5b6257084e440 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -4962,7 +4962,8 @@ private String copySegmentFiles(
         return segmentNFile;
     }
 
-    private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) {
+    // Visible for testing
+    boolean localDirectoryContains(Directory localDirectory, String file, long checksum) throws IOException {
         try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) {
             if (checksum == CodecUtil.retrieveChecksum(indexInput)) {
                 return true;
@@ -4981,6 +4982,8 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo
             logger.debug("File {} does not exist in local FS, downloading from remote store", file);
         } catch (IOException e) {
             logger.warn("Exception while reading checksum of file: {}, this can happen if file is corrupted", file);
+            // For any other exception on reading checksum, we delete the file to re-download again
+            localDirectory.deleteFile(file);
         }
         return false;
     }
diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index fa3cf7676f55c..f5f8cd1dcfb3f 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -32,6 +32,7 @@
 package org.opensearch.index.shard;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
@@ -45,6 +46,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.tests.mockfile.ExtrasFS;
 import org.apache.lucene.tests.store.BaseDirectoryWrapper;
 import org.apache.lucene.util.BytesRef;
@@ -91,6 +93,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.util.FileSystemUtils;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -163,11 +166,13 @@
 import org.junit.Assert;
 
 import java.io.IOException;
+import java.nio.channels.FileChannel;
 import java.nio.charset.Charset;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -4907,6 +4912,53 @@ public void testRecordsForceMerges() throws IOException {
         closeShards(shard);
     }
 
+    public void testLocalDirectoryContains() throws IOException {
+        IndexShard indexShard = newStartedShard(true);
+        int numDocs = between(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            indexDoc(indexShard, "_doc", Integer.toString(i));
+        }
+        flushShard(indexShard);
+        indexShard.store().incRef();
+        Directory localDirectory = indexShard.store().directory();
+        Path shardPath = indexShard.shardPath().getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
+        Path tempDir = createTempDir();
+        for (String file : localDirectory.listAll()) {
+            if (file.equals("write.lock") || file.startsWith("extra")) {
+                continue;
+            }
+            boolean corrupted = randomBoolean();
+            long checksum = 0;
+            try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) {
+                checksum = CodecUtil.retrieveChecksum(indexInput);
+            }
+            if (corrupted) {
+                Files.copy(shardPath.resolve(file), tempDir.resolve(file));
+                try (FileChannel raf = FileChannel.open(shardPath.resolve(file), StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+                    CorruptionUtils.corruptAt(shardPath.resolve(file), raf, (int) (raf.size() - 8));
+                }
+            }
+            if (corrupted == false) {
+                assertTrue(indexShard.localDirectoryContains(localDirectory, file, checksum));
+            } else {
+                assertFalse(indexShard.localDirectoryContains(localDirectory, file, checksum));
+                assertFalse(Files.exists(shardPath.resolve(file)));
+            }
+        }
+        try (Stream files = Files.list(tempDir)) {
+            files.forEach(p -> {
+                try {
+                    Files.copy(p, shardPath.resolve(p.getFileName()));
+                } catch (IOException e) {
+                    // Ignore
+                }
+            });
+        }
+        FileSystemUtils.deleteSubDirectories(tempDir);
+        indexShard.store().decRef();
+        closeShards(indexShard);
+    }
+
     private void populateSampleRemoteSegmentStats(RemoteSegmentTransferTracker tracker) {
         tracker.addUploadBytesStarted(30L);
         tracker.addUploadBytesSucceeded(10L);
diff --git a/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java b/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java
index 0dce5e78bf91f..67522bb618cf1 100644
--- a/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java
+++ b/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java
@@ -121,7 +121,7 @@ public static void corruptFile(Random random, Path... files) throws IOException
         }
     }
 
-    static void corruptAt(Path path, FileChannel channel, int position) throws IOException {
+    public static void corruptAt(Path path, FileChannel channel, int position) throws IOException {
         // read
         channel.position(position);
         long filePointer = channel.position();

From a890e518aea1a706249001c3420c1740bd1a06dd Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Wed, 25 Oct 2023 20:05:07 +0530
Subject: [PATCH 232/388] Fix exception handling for global metadata upload
 (#10889)

Signed-off-by: Dhwanil Patel 
---
 .../gateway/remote/RemoteClusterStateService.java         | 8 ++++++--
 .../gateway/remote/RemoteClusterStateServiceTests.java    | 8 +++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 329ebd0dcd2b8..57b1b972e08c0 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -369,6 +369,8 @@ public ClusterMetadataManifest writeIncrementalMetadata(
     private String writeGlobalMetadata(ClusterState clusterState) throws IOException {
 
         AtomicReference result = new AtomicReference();
+        AtomicReference exceptionReference = new AtomicReference();
+
         final BlobContainer globalMetadataContainer = globalMetadataContainer(
             clusterState.getClusterName().value(),
             clusterState.metadata().clusterUUID()
@@ -381,7 +383,7 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException
         LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> {
             logger.trace(String.format(Locale.ROOT, "GlobalMetadata uploaded successfully."));
             result.set(globalMetadataContainer.path().buildAsString() + globalMetadataFilename);
-        }, ex -> { throw new GlobalMetadataTransferException(ex.getMessage(), ex); }), latch);
+        }, ex -> { exceptionReference.set(ex); }), latch);
 
         GLOBAL_METADATA_FORMAT.writeAsyncWithUrgentPriority(
             clusterState.metadata(),
@@ -408,7 +410,9 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException
             Thread.currentThread().interrupt();
             throw exception;
         }
-
+        if (exceptionReference.get() != null) {
+            throw new GlobalMetadataTransferException(exceptionReference.get().getMessage(), exceptionReference.get());
+        }
         return result.get();
     }
 
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 5a43864f40c0c..ca88653f529f6 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -294,7 +294,13 @@ public void testWriteFullMetadataFailureForGlobalMetadata() throws IOException {
         ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
 
         doAnswer((i) -> {
-            actionListenerArgumentCaptor.getValue().onFailure(new RuntimeException("Cannot upload to remote"));
+            // For async write action listener will be called from different thread, replicating same behaviour here.
+            new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    actionListenerArgumentCaptor.getValue().onFailure(new RuntimeException("Cannot upload to remote"));
+                }
+            }).start();
             return null;
         }).when(container).asyncBlobUpload(any(WriteContext.class), actionListenerArgumentCaptor.capture());
 

From 3e64a7b92192a37546b92b811e04c4155dafbdfd Mon Sep 17 00:00:00 2001
From: Aman Khare <85096200+amkhar@users.noreply.github.com>
Date: Wed, 25 Oct 2023 21:13:52 +0530
Subject: [PATCH 233/388] Change version to 2.12.0 for cluster state stats
 (#10915)

Signed-off-by: Aman Khare 
Co-authored-by: Aman Khare 
---
 .../main/java/org/opensearch/discovery/DiscoveryStats.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java b/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java
index ea93ccd09ed39..fb341ac2ac569 100644
--- a/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java
+++ b/server/src/main/java/org/opensearch/discovery/DiscoveryStats.java
@@ -64,7 +64,7 @@ public DiscoveryStats(PendingClusterStateStats queueStats, PublishClusterStateSt
     public DiscoveryStats(StreamInput in) throws IOException {
         queueStats = in.readOptionalWriteable(PendingClusterStateStats::new);
         publishStats = in.readOptionalWriteable(PublishClusterStateStats::new);
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             clusterStateStats = in.readOptionalWriteable(ClusterStateStats::new);
         } else {
             clusterStateStats = null;
@@ -75,7 +75,7 @@ public DiscoveryStats(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(queueStats);
         out.writeOptionalWriteable(publishStats);
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(clusterStateStats);
         }
     }

From 6779633d76052585f5e20df5ac9388e14d75ffb2 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Wed, 25 Oct 2023 09:58:16 -0700
Subject: [PATCH 234/388] Fix flaky test testSendCorruptBytesToReplica (#10897)

---
 .../SegmentReplicationDisruptionIT.java       | 167 ++++++++++++++++++
 .../replication/SegmentReplicationIT.java     | 136 --------------
 2 files changed, 167 insertions(+), 136 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationDisruptionIT.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationDisruptionIT.java
new file mode 100644
index 0000000000000..66b26b5d25cfe
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationDisruptionIT.java
@@ -0,0 +1,167 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices.replication;
+
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.opensearch.action.admin.indices.recovery.RecoveryResponse;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.indices.recovery.FileChunkRequest;
+import org.opensearch.indices.recovery.RecoveryState;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.transport.MockTransportService;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportService;
+import org.junit.Before;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ * These tests simulate corruption cases during replication.  They are skipped on WindowsFS simulation where file renaming
+ * can fail with an access denied IOException because deletion is not permitted.
+ */
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+@LuceneTestCase.SuppressFileSystems("WindowsFS")
+public class SegmentReplicationDisruptionIT extends SegmentReplicationBaseIT {
+    @Before
+    private void setup() {
+        internalCluster().startClusterManagerOnlyNode();
+    }
+
+    public void testSendCorruptBytesToReplica() throws Exception {
+        final String primaryNode = internalCluster().startDataOnlyNode();
+        createIndex(
+            INDEX_NAME,
+            Settings.builder()
+                .put(indexSettings())
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put("index.refresh_interval", -1)
+                .build()
+        );
+        ensureYellow(INDEX_NAME);
+        final String replicaNode = internalCluster().startDataOnlyNode();
+        ensureGreen(INDEX_NAME);
+
+        MockTransportService primaryTransportService = ((MockTransportService) internalCluster().getInstance(
+            TransportService.class,
+            primaryNode
+        ));
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicBoolean failed = new AtomicBoolean(false);
+        primaryTransportService.addSendBehavior(
+            internalCluster().getInstance(TransportService.class, replicaNode),
+            (connection, requestId, action, request, options) -> {
+                if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK) && failed.getAndSet(true) == false) {
+                    FileChunkRequest req = (FileChunkRequest) request;
+                    TransportRequest corrupt = new FileChunkRequest(
+                        req.recoveryId(),
+                        ((FileChunkRequest) request).requestSeqNo(),
+                        ((FileChunkRequest) request).shardId(),
+                        ((FileChunkRequest) request).metadata(),
+                        ((FileChunkRequest) request).position(),
+                        new BytesArray("test"),
+                        false,
+                        0,
+                        0L
+                    );
+                    connection.sendRequest(requestId, action, corrupt, options);
+                    latch.countDown();
+                } else {
+                    connection.sendRequest(requestId, action, request, options);
+                }
+            }
+        );
+        for (int i = 0; i < 100; i++) {
+            client().prepareIndex(INDEX_NAME)
+                .setId(String.valueOf(i))
+                .setSource(jsonBuilder().startObject().field("field", i).endObject())
+                .get();
+        }
+        final long originalRecoveryTime = getRecoveryStopTime(replicaNode);
+        assertNotEquals(originalRecoveryTime, 0);
+        refresh(INDEX_NAME);
+        latch.await();
+        assertTrue(failed.get());
+        waitForNewPeerRecovery(replicaNode, originalRecoveryTime);
+        // reset checkIndex to ensure our original shard doesn't throw
+        resetCheckIndexStatus();
+        waitForSearchableDocs(100, primaryNode, replicaNode);
+    }
+
+    public void testWipeSegmentBetweenSyncs() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+        final String primaryNode = internalCluster().startDataOnlyNode();
+        createIndex(
+            INDEX_NAME,
+            Settings.builder()
+                .put(indexSettings())
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put("index.refresh_interval", -1)
+                .build()
+        );
+        ensureYellow(INDEX_NAME);
+        final String replicaNode = internalCluster().startDataOnlyNode();
+        ensureGreen(INDEX_NAME);
+
+        for (int i = 0; i < 10; i++) {
+            client().prepareIndex(INDEX_NAME)
+                .setId(String.valueOf(i))
+                .setSource(jsonBuilder().startObject().field("field", i).endObject())
+                .get();
+        }
+        refresh(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+        final long originalRecoveryTime = getRecoveryStopTime(replicaNode);
+
+        final IndexShard indexShard = getIndexShard(replicaNode, INDEX_NAME);
+        waitForSearchableDocs(INDEX_NAME, 10, List.of(replicaNode));
+        indexShard.store().directory().deleteFile("_0.si");
+
+        for (int i = 11; i < 21; i++) {
+            client().prepareIndex(INDEX_NAME)
+                .setId(String.valueOf(i))
+                .setSource(jsonBuilder().startObject().field("field", i).endObject())
+                .get();
+        }
+        refresh(INDEX_NAME);
+        waitForNewPeerRecovery(replicaNode, originalRecoveryTime);
+        resetCheckIndexStatus();
+        waitForSearchableDocs(20, primaryNode, replicaNode);
+    }
+
+    private void waitForNewPeerRecovery(String replicaNode, long originalRecoveryTime) throws Exception {
+        assertBusy(() -> {
+            // assert we have a peer recovery after the original
+            final long time = getRecoveryStopTime(replicaNode);
+            assertNotEquals(time, 0);
+            assertNotEquals(originalRecoveryTime, time);
+
+        }, 1, TimeUnit.MINUTES);
+    }
+
+    private long getRecoveryStopTime(String nodeName) {
+        final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(INDEX_NAME).get();
+        final List recoveryStates = recoveryResponse.shardRecoveryStates().get(INDEX_NAME);
+        for (RecoveryState recoveryState : recoveryStates) {
+            if (recoveryState.getTargetNode().getName().equals(nodeName)) {
+                return recoveryState.getTimer().stopTime();
+            }
+        }
+        return 0L;
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
index 81556cc270151..f48df082a25dc 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
@@ -24,7 +24,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.flush.FlushRequest;
-import org.opensearch.action.admin.indices.recovery.RecoveryResponse;
 import org.opensearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.opensearch.action.get.GetResponse;
@@ -59,7 +58,6 @@
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -73,7 +71,6 @@
 import org.opensearch.index.engine.NRTReplicationReaderManager;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.recovery.FileChunkRequest;
-import org.opensearch.indices.recovery.RecoveryState;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.node.NodeClosedException;
@@ -85,7 +82,6 @@
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.transport.MockTransportService;
-import org.opensearch.transport.TransportRequest;
 import org.opensearch.transport.TransportService;
 import org.junit.Before;
 
@@ -98,7 +94,6 @@
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 import static java.util.Arrays.asList;
@@ -1781,135 +1776,4 @@ public void testRealtimeTermVectorRequestsUnSuccessful() throws IOException {
         assertThat(response.getIndex(), equalTo(INDEX_NAME));
 
     }
-
-    public void testSendCorruptBytesToReplica() throws Exception {
-        // this test stubs transport calls specific to node-node replication.
-        assumeFalse(
-            "Skipping the test as its not compatible with segment replication with remote store.",
-            segmentReplicationWithRemoteEnabled()
-        );
-        final String primaryNode = internalCluster().startDataOnlyNode();
-        createIndex(
-            INDEX_NAME,
-            Settings.builder()
-                .put(indexSettings())
-                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
-                .put("index.refresh_interval", -1)
-                .build()
-        );
-        ensureYellow(INDEX_NAME);
-        final String replicaNode = internalCluster().startDataOnlyNode();
-        ensureGreen(INDEX_NAME);
-
-        MockTransportService primaryTransportService = ((MockTransportService) internalCluster().getInstance(
-            TransportService.class,
-            primaryNode
-        ));
-        CountDownLatch latch = new CountDownLatch(1);
-        AtomicBoolean failed = new AtomicBoolean(false);
-        primaryTransportService.addSendBehavior(
-            internalCluster().getInstance(TransportService.class, replicaNode),
-            (connection, requestId, action, request, options) -> {
-                if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK) && failed.getAndSet(true) == false) {
-                    FileChunkRequest req = (FileChunkRequest) request;
-                    logger.info("SENDING CORRUPT file chunk [{}] lastChunk: {}", req, req.lastChunk());
-                    TransportRequest corrupt = new FileChunkRequest(
-                        req.recoveryId(),
-                        ((FileChunkRequest) request).requestSeqNo(),
-                        ((FileChunkRequest) request).shardId(),
-                        ((FileChunkRequest) request).metadata(),
-                        ((FileChunkRequest) request).position(),
-                        new BytesArray("test"),
-                        false,
-                        0,
-                        0L
-                    );
-                    connection.sendRequest(requestId, action, corrupt, options);
-                    latch.countDown();
-                } else {
-                    connection.sendRequest(requestId, action, request, options);
-                }
-            }
-        );
-        for (int i = 0; i < 100; i++) {
-            client().prepareIndex(INDEX_NAME)
-                .setId(String.valueOf(i))
-                .setSource(jsonBuilder().startObject().field("field", i).endObject())
-                .get();
-        }
-        final long originalRecoveryTime = getRecoveryStopTime(replicaNode);
-        assertNotEquals(originalRecoveryTime, 0);
-        refresh(INDEX_NAME);
-        latch.await();
-        assertTrue(failed.get());
-        waitForNewPeerRecovery(replicaNode, originalRecoveryTime);
-        // reset checkIndex to ensure our original shard doesn't throw
-        resetCheckIndexStatus();
-        waitForSearchableDocs(100, primaryNode, replicaNode);
-    }
-
-    public void testWipeSegmentBetweenSyncs() throws Exception {
-        internalCluster().startClusterManagerOnlyNode();
-        final String primaryNode = internalCluster().startDataOnlyNode();
-        createIndex(
-            INDEX_NAME,
-            Settings.builder()
-                .put(indexSettings())
-                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
-                .put("index.refresh_interval", -1)
-                .build()
-        );
-        ensureYellow(INDEX_NAME);
-        final String replicaNode = internalCluster().startDataOnlyNode();
-        ensureGreen(INDEX_NAME);
-
-        for (int i = 0; i < 10; i++) {
-            client().prepareIndex(INDEX_NAME)
-                .setId(String.valueOf(i))
-                .setSource(jsonBuilder().startObject().field("field", i).endObject())
-                .get();
-        }
-        refresh(INDEX_NAME);
-        ensureGreen(INDEX_NAME);
-        final long originalRecoveryTime = getRecoveryStopTime(replicaNode);
-
-        final IndexShard indexShard = getIndexShard(replicaNode, INDEX_NAME);
-        waitForSearchableDocs(INDEX_NAME, 10, List.of(replicaNode));
-        indexShard.store().directory().deleteFile("_0.si");
-
-        for (int i = 11; i < 21; i++) {
-            client().prepareIndex(INDEX_NAME)
-                .setId(String.valueOf(i))
-                .setSource(jsonBuilder().startObject().field("field", i).endObject())
-                .get();
-        }
-        refresh(INDEX_NAME);
-        waitForNewPeerRecovery(replicaNode, originalRecoveryTime);
-        resetCheckIndexStatus();
-        waitForSearchableDocs(20, primaryNode, replicaNode);
-    }
-
-    private void waitForNewPeerRecovery(String replicaNode, long originalRecoveryTime) throws Exception {
-        assertBusy(() -> {
-            // assert we have a peer recovery after the original
-            final long time = getRecoveryStopTime(replicaNode);
-            assertNotEquals(time, 0);
-            assertNotEquals(originalRecoveryTime, time);
-
-        }, 1, TimeUnit.MINUTES);
-    }
-
-    private long getRecoveryStopTime(String nodeName) {
-        final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(INDEX_NAME).get();
-        final List recoveryStates = recoveryResponse.shardRecoveryStates().get(INDEX_NAME);
-        logger.info("Recovery states {}", recoveryResponse);
-        for (RecoveryState recoveryState : recoveryStates) {
-            if (recoveryState.getTargetNode().getName().equals(nodeName)) {
-                return recoveryState.getTimer().stopTime();
-            }
-        }
-        return 0L;
-    }
 }

From 44a9f180a5643e3b4dee1150c5a68d1591270ced Mon Sep 17 00:00:00 2001
From: Aman Khare <85096200+amkhar@users.noreply.github.com>
Date: Thu, 26 Oct 2023 00:21:12 +0530
Subject: [PATCH 235/388] Fix flaky testClusterStateBatchedUpdates test
 (#10922)

Signed-off-by: Aman Khare 
Co-authored-by: Aman Khare 
---
 .../org/opensearch/cluster/service/MasterServiceTests.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
index 4c0ca826f5dcc..85f6c129944fa 100644
--- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
@@ -487,6 +487,9 @@ public void onFailure(String source, Exception e) {
                     }
                 });
                 assertBusy(mockAppender::assertAllExpectationsMatched);
+                // verify stats values after state is published
+                assertEquals(1, clusterManagerService.getClusterStateStats().getUpdateSuccess());
+                assertEquals(0, clusterManagerService.getClusterStateStats().getUpdateFailed());
             }
         }
     }
@@ -691,9 +694,6 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
                     submittedTasksPerThread.get(entry.getKey()).get()
                 );
             }
-            // verify stats values after state is published
-            assertEquals(1, clusterManagerService.getClusterStateStats().getUpdateSuccess());
-            assertEquals(0, clusterManagerService.getClusterStateStats().getUpdateFailed());
         }
     }
 

From fb6fe1bf4518a4a4a8507564fbd4db1971f29151 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Wed, 25 Oct 2023 14:20:48 -0700
Subject: [PATCH 236/388] Fix flaky test
 IndexShardTests.testLocalDirectoryContains (#10929)

This test is breaking for WindowsFS only. Moving it to a separate file where it is skipped on WindowsFS.

Signed-off-by: Marc Handalian 
---
 .../index/shard/IndexShardTests.java          | 52 -------------
 .../RemoteIndexShardCorruptionTests.java      | 75 +++++++++++++++++++
 2 files changed, 75 insertions(+), 52 deletions(-)
 create mode 100644 server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java

diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index f5f8cd1dcfb3f..fa3cf7676f55c 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -32,7 +32,6 @@
 package org.opensearch.index.shard;
 
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
@@ -46,7 +45,6 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.tests.mockfile.ExtrasFS;
 import org.apache.lucene.tests.store.BaseDirectoryWrapper;
 import org.apache.lucene.util.BytesRef;
@@ -93,7 +91,6 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
-import org.opensearch.core.util.FileSystemUtils;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -166,13 +163,11 @@
 import org.junit.Assert;
 
 import java.io.IOException;
-import java.nio.channels.FileChannel;
 import java.nio.charset.Charset;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
-import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -4912,53 +4907,6 @@ public void testRecordsForceMerges() throws IOException {
         closeShards(shard);
     }
 
-    public void testLocalDirectoryContains() throws IOException {
-        IndexShard indexShard = newStartedShard(true);
-        int numDocs = between(1, 10);
-        for (int i = 0; i < numDocs; i++) {
-            indexDoc(indexShard, "_doc", Integer.toString(i));
-        }
-        flushShard(indexShard);
-        indexShard.store().incRef();
-        Directory localDirectory = indexShard.store().directory();
-        Path shardPath = indexShard.shardPath().getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
-        Path tempDir = createTempDir();
-        for (String file : localDirectory.listAll()) {
-            if (file.equals("write.lock") || file.startsWith("extra")) {
-                continue;
-            }
-            boolean corrupted = randomBoolean();
-            long checksum = 0;
-            try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) {
-                checksum = CodecUtil.retrieveChecksum(indexInput);
-            }
-            if (corrupted) {
-                Files.copy(shardPath.resolve(file), tempDir.resolve(file));
-                try (FileChannel raf = FileChannel.open(shardPath.resolve(file), StandardOpenOption.READ, StandardOpenOption.WRITE)) {
-                    CorruptionUtils.corruptAt(shardPath.resolve(file), raf, (int) (raf.size() - 8));
-                }
-            }
-            if (corrupted == false) {
-                assertTrue(indexShard.localDirectoryContains(localDirectory, file, checksum));
-            } else {
-                assertFalse(indexShard.localDirectoryContains(localDirectory, file, checksum));
-                assertFalse(Files.exists(shardPath.resolve(file)));
-            }
-        }
-        try (Stream files = Files.list(tempDir)) {
-            files.forEach(p -> {
-                try {
-                    Files.copy(p, shardPath.resolve(p.getFileName()));
-                } catch (IOException e) {
-                    // Ignore
-                }
-            });
-        }
-        FileSystemUtils.deleteSubDirectories(tempDir);
-        indexShard.store().decRef();
-        closeShards(indexShard);
-    }
-
     private void populateSampleRemoteSegmentStats(RemoteSegmentTransferTracker tracker) {
         tracker.addUploadBytesStarted(30L);
         tracker.addUploadBytesSucceeded(10L);
diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java
new file mode 100644
index 0000000000000..21bf580712761
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java
@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.shard;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.opensearch.core.util.FileSystemUtils;
+import org.opensearch.test.CorruptionUtils;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.stream.Stream;
+
+@LuceneTestCase.SuppressFileSystems("WindowsFS")
+public class RemoteIndexShardCorruptionTests extends IndexShardTestCase {
+
+    public void testLocalDirectoryContains() throws IOException {
+        IndexShard indexShard = newStartedShard(true);
+        int numDocs = between(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            indexDoc(indexShard, "_doc", Integer.toString(i));
+        }
+        flushShard(indexShard);
+        indexShard.store().incRef();
+        Directory localDirectory = indexShard.store().directory();
+        Path shardPath = indexShard.shardPath().getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
+        Path tempDir = createTempDir();
+        for (String file : localDirectory.listAll()) {
+            if (file.equals("write.lock") || file.startsWith("extra")) {
+                continue;
+            }
+            boolean corrupted = randomBoolean();
+            long checksum = 0;
+            try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) {
+                checksum = CodecUtil.retrieveChecksum(indexInput);
+            }
+            if (corrupted) {
+                Files.copy(shardPath.resolve(file), tempDir.resolve(file));
+                try (FileChannel raf = FileChannel.open(shardPath.resolve(file), StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+                    CorruptionUtils.corruptAt(shardPath.resolve(file), raf, (int) (raf.size() - 8));
+                }
+            }
+            if (corrupted == false) {
+                assertTrue(indexShard.localDirectoryContains(localDirectory, file, checksum));
+            } else {
+                assertFalse(indexShard.localDirectoryContains(localDirectory, file, checksum));
+                assertFalse(Files.exists(shardPath.resolve(file)));
+            }
+        }
+        try (Stream files = Files.list(tempDir)) {
+            files.forEach(p -> {
+                try {
+                    Files.copy(p, shardPath.resolve(p.getFileName()));
+                } catch (IOException e) {
+                    // Ignore
+                }
+            });
+        }
+        FileSystemUtils.deleteSubDirectories(tempDir);
+        indexShard.store().decRef();
+        closeShards(indexShard);
+    }
+}

From b17d4a8d6fa2463adbf5fc7fbaa1b1bcc4424121 Mon Sep 17 00:00:00 2001
From: Ashish 
Date: Thu, 26 Oct 2023 06:20:41 +0530
Subject: [PATCH 237/388] [Remote Store] Fix refresh lag bug on primary term
 change (#10918)

* [Remote Store] Fix refresh lag bug on primary term change

Signed-off-by: Ashish Singh 

* Add Integ Tests

Signed-off-by: Ashish Singh 

* Incorporate PR review feedback

Signed-off-by: Ashish Singh 

* Empty-Commit

Signed-off-by: Ashish Singh 

---------

Signed-off-by: Ashish Singh 
---
 .../remotestore/RemoteStoreStatsIT.java       | 23 ++++++++++++++++
 .../shard/RemoteStoreRefreshListener.java     | 27 ++++++++++++-------
 2 files changed, 41 insertions(+), 9 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
index b1dbb0a900bc7..2d3ab135d0377 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java
@@ -655,6 +655,29 @@ public void testStatsCorrectnessOnFailover() {
         logger.info("Test completed");
     }
 
+    public void testZeroLagOnCreateIndex() throws InterruptedException {
+        setup();
+        String clusterManagerNode = internalCluster().getClusterManagerName();
+
+        int numOfShards = randomIntBetween(1, 3);
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(1, numOfShards));
+        ensureGreen(INDEX_NAME);
+        long currentTimeNs = System.nanoTime();
+        while (currentTimeNs == System.nanoTime()) {
+            Thread.sleep(10);
+        }
+
+        for (int i = 0; i < numOfShards; i++) {
+            RemoteStoreStatsResponse response = client(clusterManagerNode).admin()
+                .cluster()
+                .prepareRemoteStoreStats(INDEX_NAME, String.valueOf(i))
+                .get();
+            for (RemoteStoreStats remoteStoreStats : response.getRemoteStoreStats()) {
+                assertEquals(0, remoteStoreStats.getSegmentStats().refreshTimeLagMs);
+            }
+        }
+    }
+
     private void indexDocs() {
         for (int i = 0; i < randomIntBetween(5, 10); i++) {
             if (randomBoolean()) {
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index 3e97b07abfb5d..464adc88ae16f 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -86,7 +86,7 @@ public final class RemoteStoreRefreshListener extends CloseableRetryableRefreshL
     private final RemoteSegmentStoreDirectory remoteDirectory;
     private final RemoteSegmentTransferTracker segmentTracker;
     private final Map localSegmentChecksumMap;
-    private long primaryTerm;
+    private volatile long primaryTerm;
     private volatile Iterator backoffDelayIterator;
     private final SegmentReplicationCheckpointPublisher checkpointPublisher;
 
@@ -126,10 +126,9 @@ protected void runAfterRefreshExactlyOnce(boolean didRefresh) {
         // We have 2 separate methods to check if sync needs to be done or not. This is required since we use the return boolean
         // from isReadyForUpload to schedule refresh retries as the index shard or the primary mode are not in complete
         // ready state.
-        if (shouldSync(didRefresh) && isReadyForUpload()) {
-            segmentTracker.updateLocalRefreshTimeAndSeqNo();
+        if (shouldSync(didRefresh, true) && isReadyForUpload()) {
             try {
-                initializeRemoteDirectoryOnTermUpdate();
+                segmentTracker.updateLocalRefreshTimeAndSeqNo();
                 try (GatedCloseable segmentInfosGatedCloseable = indexShard.getSegmentInfosSnapshot()) {
                     Collection localSegmentsPostRefresh = segmentInfosGatedCloseable.get().files(true);
                     updateLocalSizeMapAndTracker(localSegmentsPostRefresh);
@@ -150,7 +149,7 @@ protected void runAfterRefreshExactlyOnce(boolean didRefresh) {
     @Override
     protected boolean performAfterRefreshWithPermit(boolean didRefresh) {
         boolean successful;
-        if (shouldSync(didRefresh)) {
+        if (shouldSync(didRefresh, false)) {
             successful = syncSegments();
         } else {
             successful = true;
@@ -158,10 +157,15 @@ protected boolean performAfterRefreshWithPermit(boolean didRefresh) {
         return successful;
     }
 
-    private boolean shouldSync(boolean didRefresh) {
-        return this.primaryTerm != indexShard.getOperationPrimaryTerm()
-            // If the readers change, didRefresh is always true.
-            || didRefresh
+    /**
+     * This checks if there is a sync required to remote.
+     *
+     * @param didRefresh             if the readers changed.
+     * @param skipPrimaryTermCheck consider change in primary term or not for should sync
+     * @return true if sync is needed
+     */
+    private boolean shouldSync(boolean didRefresh, boolean skipPrimaryTermCheck) {
+        boolean shouldSync = didRefresh // If the readers change, didRefresh is always true.
             // The third condition exists for uploading the zero state segments where the refresh has not changed the reader
             // reference, but it is important to upload the zero state segments so that the restore does not break.
             || remoteDirectory.getSegmentsUploadedToRemoteStore().isEmpty()
@@ -169,6 +173,10 @@ private boolean shouldSync(boolean didRefresh) {
             // we update the primary term and the same condition would not evaluate to true again in syncSegments.
             // Below check ensures that if there is commit, then that gets picked up by both 1st and 2nd shouldSync call.
             || isRefreshAfterCommitSafe();
+        if (shouldSync || skipPrimaryTermCheck) {
+            return shouldSync;
+        }
+        return this.primaryTerm != indexShard.getOperationPrimaryTerm();
     }
 
     private boolean syncSegments() {
@@ -188,6 +196,7 @@ private boolean syncSegments() {
 
         try {
             try {
+                initializeRemoteDirectoryOnTermUpdate();
                 // if a new segments_N file is present in local that is not uploaded to remote store yet, it
                 // is considered as a first refresh post commit. A cleanup of stale commit files is triggered.
                 // This is done to avoid delete post each refresh.

From 003b2cf30103282b031f4462ed53488edb60b9a4 Mon Sep 17 00:00:00 2001
From: Suraj Singh 
Date: Wed, 25 Oct 2023 18:22:19 -0700
Subject: [PATCH 238/388] [Segment Replication] Handle exceptions on local file
 read during replication (#10933)

* Handle exceptions on file read

Signed-off-by: Suraj Singh 

* Address review comments

Signed-off-by: Suraj Singh 

---------

Signed-off-by: Suraj Singh 
---
 .../replication/SegmentReplicationTarget.java |  11 +-
 .../index/shard/RemoteIndexShardTests.java    | 159 ++++++++++++++----
 2 files changed, 134 insertions(+), 36 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java
index cd6dbe8af90d9..cc71ef816e525 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java
@@ -232,6 +232,7 @@ private List getFiles(CheckpointInfoResponse checkpointInfo)
         return missingFiles;
     }
 
+    // pkg private for tests
     private boolean validateLocalChecksum(StoreFileMetadata file) {
         try (IndexInput indexInput = indexShard.store().directory().openInput(file.name(), IOContext.DEFAULT)) {
             String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput));
@@ -243,7 +244,15 @@ private boolean validateLocalChecksum(StoreFileMetadata file) {
                 return false;
             }
         } catch (IOException e) {
-            throw new UncheckedIOException("Error reading " + file, e);
+            logger.warn("Error reading " + file, e);
+            // Delete file on exceptions so that it can be re-downloaded. This is safe to do as this file is local only
+            // and not referenced by reader.
+            try {
+                indexShard.store().directory().deleteFile(file.name());
+            } catch (IOException ex) {
+                throw new UncheckedIOException("Error reading " + file, e);
+            }
+            return false;
         }
     }
 
diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
index 703a7d457d5b6..2ce0bdc607189 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
@@ -31,18 +31,20 @@
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 import org.opensearch.indices.replication.common.ReplicationFailedException;
 import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.test.CorruptionUtils;
 import org.hamcrest.MatcherAssert;
 import org.junit.Assert;
 
 import java.io.IOException;
+import java.nio.channels.FileChannel;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
@@ -371,37 +373,9 @@ public void testSegRepSucceedsOnPreviousCopiedFiles() throws Exception {
 
             final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class);
             final SegmentReplicationTargetService targetService = newTargetService(sourceFactory);
-            Runnable[] runAfterGetFiles = { () -> { throw new RuntimeException("Simulated"); }, () -> {} };
-            AtomicInteger index = new AtomicInteger(0);
-            RemoteStoreReplicationSource testRSReplicationSource = new RemoteStoreReplicationSource(replica) {
-                @Override
-                public void getCheckpointMetadata(
-                    long replicationId,
-                    ReplicationCheckpoint checkpoint,
-                    ActionListener listener
-                ) {
-                    super.getCheckpointMetadata(replicationId, checkpoint, listener);
-                }
-
-                @Override
-                public void getSegmentFiles(
-                    long replicationId,
-                    ReplicationCheckpoint checkpoint,
-                    List filesToFetch,
-                    IndexShard indexShard,
-                    BiConsumer fileProgressTracker,
-                    ActionListener listener
-                ) {
-                    super.getSegmentFiles(replicationId, checkpoint, filesToFetch, indexShard, (fileName, bytesRecovered) -> {}, listener);
-                    runAfterGetFiles[index.getAndIncrement()].run();
-                }
-
-                @Override
-                public String getDescription() {
-                    return "TestRemoteStoreReplicationSource";
-                }
-            };
-            when(sourceFactory.get(any())).thenReturn(testRSReplicationSource);
+            when(sourceFactory.get(any())).thenReturn(
+                getRemoteStoreReplicationSource(replica, () -> { throw new RuntimeException("Simulated"); })
+            );
             CountDownLatch latch = new CountDownLatch(1);
 
             // Start first round of segment replication. This should fail with simulated error but with replica having
@@ -412,6 +386,7 @@ public String getDescription() {
                 new SegmentReplicationTargetService.SegmentReplicationListener() {
                     @Override
                     public void onReplicationDone(SegmentReplicationState state) {
+                        latch.countDown();
                         Assert.fail("Replication should fail with simulated error");
                     }
 
@@ -421,9 +396,9 @@ public void onReplicationFailure(
                         ReplicationFailedException e,
                         boolean sendShardFailure
                     ) {
+                        latch.countDown();
                         assertFalse(sendShardFailure);
                         logger.error("Replication error", e);
-                        latch.countDown();
                     }
                 }
             );
@@ -439,7 +414,8 @@ public void onReplicationFailure(
             assertEquals("Files should be copied to disk", false, onDiskFiles.isEmpty());
             assertEquals(target.state().getStage(), SegmentReplicationState.Stage.GET_FILES);
 
-            // Start next round of segment replication
+            // Start next round of segment replication and not throwing exception resulting in commit on replica
+            when(sourceFactory.get(any())).thenReturn(getRemoteStoreReplicationSource(replica, () -> {}));
             CountDownLatch waitForSecondRound = new CountDownLatch(1);
             final SegmentReplicationTarget newTarget = targetService.startReplication(
                 replica,
@@ -456,9 +432,9 @@ public void onReplicationFailure(
                         ReplicationFailedException e,
                         boolean sendShardFailure
                     ) {
+                        waitForSecondRound.countDown();
                         logger.error("Replication error", e);
                         Assert.fail("Replication should not fail");
-                        waitForSecondRound.countDown();
                     }
                 }
             );
@@ -471,6 +447,119 @@ public void onReplicationFailure(
         }
     }
 
+    /**
+     * This test validates that local non-readable (corrupt, partially) on disk are deleted vs failing the
+     * replication event. This test mimics local files (not referenced by reader) by throwing exception post file copy and
+     * blocking update of reader. Once this is done, it corrupts one segment file and ensure that file is deleted in next
+     * round of segment replication by ensuring doc count.
+     */
+    public void testNoFailuresOnFileReads() throws Exception {
+        try (ReplicationGroup shards = createGroup(1, getIndexSettings(), new NRTReplicationEngineFactory())) {
+            shards.startAll();
+            IndexShard primary = shards.getPrimary();
+            final IndexShard replica = shards.getReplicas().get(0);
+
+            final int docCount = 10;
+            shards.indexDocs(docCount);
+            primary.refresh("Test");
+
+            final SegmentReplicationSourceFactory sourceFactory = mock(SegmentReplicationSourceFactory.class);
+            final SegmentReplicationTargetService targetService = newTargetService(sourceFactory);
+            when(sourceFactory.get(any())).thenReturn(
+                getRemoteStoreReplicationSource(replica, () -> { throw new RuntimeException("Simulated"); })
+            );
+            CountDownLatch waitOnReplicationCompletion = new CountDownLatch(1);
+
+            // Start first round of segment replication. This should fail with simulated error but with replica having
+            // files in its local store but not in active reader.
+            SegmentReplicationTarget segmentReplicationTarget = targetService.startReplication(
+                replica,
+                primary.getLatestReplicationCheckpoint(),
+                new SegmentReplicationTargetService.SegmentReplicationListener() {
+                    @Override
+                    public void onReplicationDone(SegmentReplicationState state) {
+                        waitOnReplicationCompletion.countDown();
+                        Assert.fail("Replication should fail with simulated error");
+                    }
+
+                    @Override
+                    public void onReplicationFailure(
+                        SegmentReplicationState state,
+                        ReplicationFailedException e,
+                        boolean sendShardFailure
+                    ) {
+                        waitOnReplicationCompletion.countDown();
+                        assertFalse(sendShardFailure);
+                    }
+                }
+            );
+            waitOnReplicationCompletion.await();
+            assertBusy(() -> { assertEquals("Target should be closed", 0, segmentReplicationTarget.refCount()); });
+            String fileToCorrupt = null;
+            // Corrupt one data file
+            Path shardPath = replica.shardPath().getDataPath().resolve(ShardPath.INDEX_FOLDER_NAME);
+            for (String file : replica.store().directory().listAll()) {
+                if (file.equals("write.lock") || file.startsWith("extra") || file.startsWith("segment")) {
+                    continue;
+                }
+                fileToCorrupt = file;
+                logger.info("--> Corrupting file {}", fileToCorrupt);
+                try (FileChannel raf = FileChannel.open(shardPath.resolve(file), StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+                    CorruptionUtils.corruptAt(shardPath.resolve(file), raf, (int) (raf.size() - 8));
+                }
+                break;
+            }
+            Assert.assertNotNull(fileToCorrupt);
+
+            // Ingest more data and start next round of segment replication
+            shards.indexDocs(docCount);
+            primary.refresh("Post corruption");
+            replicateSegments(primary, List.of(replica));
+
+            assertDocCount(primary, 2 * docCount);
+            assertDocCount(replica, 2 * docCount);
+
+            final Store.RecoveryDiff diff = Store.segmentReplicationDiff(primary.getSegmentMetadataMap(), replica.getSegmentMetadataMap());
+            assertTrue(diff.missing.isEmpty());
+            assertTrue(diff.different.isEmpty());
+
+            // clean up
+            shards.removeReplica(replica);
+            closeShards(replica);
+        }
+    }
+
+    private RemoteStoreReplicationSource getRemoteStoreReplicationSource(IndexShard shard, Runnable postGetFilesRunnable) {
+        return new RemoteStoreReplicationSource(shard) {
+            @Override
+            public void getCheckpointMetadata(
+                long replicationId,
+                ReplicationCheckpoint checkpoint,
+                ActionListener listener
+            ) {
+                super.getCheckpointMetadata(replicationId, checkpoint, listener);
+            }
+
+            @Override
+            public void getSegmentFiles(
+                long replicationId,
+                ReplicationCheckpoint checkpoint,
+                List filesToFetch,
+                IndexShard indexShard,
+                BiConsumer fileProgressTracker,
+                ActionListener listener
+            ) {
+                super.getSegmentFiles(replicationId, checkpoint, filesToFetch, indexShard, (fileName, bytesRecovered) -> {}, listener);
+                postGetFilesRunnable.run();
+            }
+
+            @Override
+            public String getDescription() {
+                return "TestRemoteStoreReplicationSource";
+            }
+        };
+    }
+
     @Override
     protected void validateShardIdleWithNoReplicas(IndexShard primary) {
         // ensure search idle conditions are met.

From fe8b2d545da09e3d80f74cd57a7ae991e1005abe Mon Sep 17 00:00:00 2001
From: Ashish 
Date: Thu, 26 Oct 2023 07:23:18 +0530
Subject: [PATCH 239/388] [Remote Store] Fix shard failure on flush due to
 upload timeout (#10926)

---
 .../transfer/TranslogTransferManager.java     | 10 +-
 .../TranslogTransferManagerTests.java         | 94 +++++++++++++++++++
 2 files changed, 100 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
index ece6f6d5a534f..2f6055df87804 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
@@ -42,7 +42,6 @@
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
 import static org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot;
@@ -156,14 +155,17 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
 
             try {
                 if (latch.await(TRANSFER_TIMEOUT_IN_MILLIS, TimeUnit.MILLISECONDS) == false) {
-                    Exception ex = new TimeoutException("Timed out waiting for transfer of snapshot " + transferSnapshot + " to complete");
+                    Exception ex = new TranslogUploadFailedException(
+                        "Timed out waiting for transfer of snapshot " + transferSnapshot + " to complete"
+                    );
                     exceptionList.forEach(ex::addSuppressed);
                     throw ex;
                 }
             } catch (InterruptedException ex) {
-                exceptionList.forEach(ex::addSuppressed);
+                Exception exception = new TranslogUploadFailedException("Failed to upload " + transferSnapshot, ex);
+                exceptionList.forEach(exception::addSuppressed);
                 Thread.currentThread().interrupt();
-                throw ex;
+                throw exception;
             }
             if (exceptionList.isEmpty()) {
                 TransferFileSnapshot tlogMetadata = prepareMetadata(transferSnapshot);
diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
index af596e7df02c2..e34bc078896f9 100644
--- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.action.LatchedActionListener;
+import org.opensearch.common.SetOnce;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobMetadata;
 import org.opensearch.common.blobstore.BlobPath;
@@ -35,6 +36,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedList;
@@ -180,6 +182,93 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) {
         assertEquals(4, fileTransferTracker.allUploaded().size());
     }
 
+    public void testTransferSnapshotOnUploadTimeout() throws Exception {
+        doAnswer(invocationOnMock -> {
+            Thread.sleep(31 * 1000);
+            return null;
+        }).when(transferService).uploadBlobs(anySet(), anyMap(), any(ActionListener.class), any(WritePriority.class));
+        FileTransferTracker fileTransferTracker = new FileTransferTracker(
+            new ShardId("index", "indexUUid", 0),
+            remoteTranslogTransferTracker
+        );
+        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
+            shardId,
+            transferService,
+            remoteBaseTransferPath,
+            fileTransferTracker,
+            remoteTranslogTransferTracker
+        );
+        SetOnce exception = new SetOnce<>();
+        translogTransferManager.transferSnapshot(createTransferSnapshot(), new TranslogTransferListener() {
+            @Override
+            public void onUploadComplete(TransferSnapshot transferSnapshot) {}
+
+            @Override
+            public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) {
+                exception.set(ex);
+            }
+        });
+        assertNotNull(exception.get());
+        assertTrue(exception.get() instanceof TranslogUploadFailedException);
+        assertEquals("Timed out waiting for transfer of snapshot test-to-string to complete", exception.get().getMessage());
+    }
+
+    public void testTransferSnapshotOnThreadInterrupt() throws Exception {
+        SetOnce uploadThread = new SetOnce<>();
+        doAnswer(invocationOnMock -> {
+            uploadThread.set(new Thread(() -> {
+                ActionListener listener = invocationOnMock.getArgument(2);
+                try {
+                    Thread.sleep(31 * 1000);
+                } catch (InterruptedException ignore) {
+                    List list = new ArrayList<>(invocationOnMock.getArgument(0));
+                    listener.onFailure(new FileTransferException(list.get(0), ignore));
+                }
+            }));
+            uploadThread.get().start();
+            return null;
+        }).when(transferService).uploadBlobs(anySet(), anyMap(), any(ActionListener.class), any(WritePriority.class));
+        FileTransferTracker fileTransferTracker = new FileTransferTracker(
+            new ShardId("index", "indexUUid", 0),
+            remoteTranslogTransferTracker
+        );
+        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
+            shardId,
+            transferService,
+            remoteBaseTransferPath,
+            fileTransferTracker,
+            remoteTranslogTransferTracker
+        );
+        SetOnce exception = new SetOnce<>();
+
+        Thread thread = new Thread(() -> {
+            try {
+                translogTransferManager.transferSnapshot(createTransferSnapshot(), new TranslogTransferListener() {
+                    @Override
+                    public void onUploadComplete(TransferSnapshot transferSnapshot) {}
+
+                    @Override
+                    public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) {
+                        exception.set(ex);
+                    }
+                });
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        thread.start();
+
+        Thread.sleep(1000);
+        // Interrupt the thread
+        thread.interrupt();
+        assertBusy(() -> {
+            assertNotNull(exception.get());
+            assertTrue(exception.get() instanceof TranslogUploadFailedException);
+            assertEquals("Failed to upload test-to-string", exception.get().getMessage());
+        });
+        uploadThread.get().interrupt();
+    }
+
     private TransferSnapshot createTransferSnapshot() {
         return new TransferSnapshot() {
             @Override
@@ -232,6 +321,11 @@ public Set getTranslogFileSnapshots() {
             public TranslogTransferMetadata getTranslogTransferMetadata() {
                 return new TranslogTransferMetadata(primaryTerm, generation, minTranslogGeneration, randomInt(5));
             }
+
+            @Override
+            public String toString() {
+                return "test-to-string";
+            }
         };
     }
 

From d1c94b524ea54f3fce3e40e211bb7108800c07de Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Thu, 26 Oct 2023 11:35:37 +0530
Subject: [PATCH 240/388] Improved logging around remote cluster state (#10892)

* Improved logging around remote cluster state

Signed-off-by: Dhwanil Patel 
---
 .../remote/RemoteClusterStateService.java     | 28 +++++++++++++------
 .../recovery/RemoteStoreRestoreService.java   |  1 +
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 57b1b972e08c0..fa4eef79c46dc 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -244,9 +244,8 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri
                 allUploadedIndexMetadata.size()
             );
         } else {
-            // todo change to debug
             logger.info(
-                "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices",
+                "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices and global metadata",
                 durationMillis,
                 allUploadedIndexMetadata.size()
             );
@@ -285,6 +284,7 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         if (updateGlobalMetadata || previousManifest.getGlobalMetadataFileName() == null) {
             globalMetadataFile = writeGlobalMetadata(clusterState);
         } else {
+            logger.debug("Global metadata has not updated in cluster state, skipping upload of it");
             globalMetadataFile = previousManifest.getGlobalMetadataFileName();
         }
 
@@ -305,7 +305,7 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         for (final IndexMetadata indexMetadata : clusterState.metadata().indices().values()) {
             final Long previousVersion = previousStateIndexMetadataVersionByName.get(indexMetadata.getIndex().getName());
             if (previousVersion == null || indexMetadata.getVersion() != previousVersion) {
-                logger.trace(
+                logger.debug(
                     "updating metadata for [{}], changing version from [{}] to [{}]",
                     indexMetadata.getIndex(),
                     previousVersion,
@@ -342,18 +342,22 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         if (durationMillis >= slowWriteLoggingThreshold.getMillis()) {
             logger.warn(
                 "writing cluster state took [{}ms] which is above the warn threshold of [{}]; "
-                    + "wrote  metadata for [{}] indices and skipped [{}] unchanged indices",
+                    + "wrote  metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]",
                 durationMillis,
                 slowWriteLoggingThreshold,
                 numIndicesUpdated,
-                numIndicesUnchanged
+                numIndicesUnchanged,
+                updateGlobalMetadata
             );
         } else {
-            logger.trace(
-                "writing cluster state took [{}ms]; " + "wrote metadata for [{}] indices and skipped [{}] unchanged indices",
+            logger.info(
+                "writing cluster state for version [{}] took [{}ms]; "
+                    + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]",
+                manifest.getStateVersion(),
                 durationMillis,
                 numIndicesUpdated,
-                numIndicesUnchanged
+                numIndicesUnchanged,
+                updateGlobalMetadata
             );
         }
         return manifest;
@@ -605,6 +609,11 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust
             blobStoreRepository.getCompressor(),
             FORMAT_PARAMS
         );
+        logger.debug(
+            "Metadata manifest file [{}] written during [{}] phase. ",
+            fileName,
+            uploadManifest.isCommitted() ? "commit" : "publish"
+        );
     }
 
     private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) {
@@ -912,6 +921,7 @@ private List createClusterChain(final Map {
                     indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));

From 746ca09ac3ce33b27e94c871a94f0ded076e19e3 Mon Sep 17 00:00:00 2001
From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com>
Date: Thu, 26 Oct 2023 11:55:13 +0530
Subject: [PATCH 241/388] Fix valid cluster UUID logic for uncommitted cluster
 UUIDs (#10916)

Signed-off-by: Sooraj Sinha 
---
 .../remote/RemoteClusterStateService.java     | 36 +++++++-------
 .../RemoteClusterStateServiceTests.java       | 49 +++++++++++++++----
 2 files changed, 59 insertions(+), 26 deletions(-)

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index fa4eef79c46dc..b3309b1fd8a63 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -881,25 +881,31 @@ private Map getLatestManifestForAllClusterUUIDs
      * @return List of cluster UUIDs. The first element is the most recent cluster UUID in the chain
      */
     private List createClusterChain(final Map manifestsByClusterUUID, final String clusterName) {
-        final Map clusterUUIDGraph = manifestsByClusterUUID.values()
+        final List validClusterManifests = manifestsByClusterUUID.values()
             .stream()
+            .filter(this::isValidClusterUUID)
+            .collect(Collectors.toList());
+        final Map clusterUUIDGraph = validClusterManifests.stream()
             .collect(Collectors.toMap(ClusterMetadataManifest::getClusterUUID, ClusterMetadataManifest::getPreviousClusterUUID));
-        final List validClusterUUIDs = manifestsByClusterUUID.values()
-            .stream()
-            .filter(m -> !isInvalidClusterUUID(m) && !clusterUUIDGraph.containsValue(m.getClusterUUID()))
+        final List topLevelClusterUUIDs = validClusterManifests.stream()
             .map(ClusterMetadataManifest::getClusterUUID)
+            .filter(clusterUUID -> !clusterUUIDGraph.containsValue(clusterUUID))
             .collect(Collectors.toList());
-        if (validClusterUUIDs.isEmpty()) {
-            logger.info("There is no valid previous cluster UUID");
+
+        if (topLevelClusterUUIDs.isEmpty()) {
+            // This can occur only when there are no valid cluster UUIDs
+            assert validClusterManifests.isEmpty() : "There are no top level cluster UUIDs even when there are valid cluster UUIDs";
+            logger.info("There is no valid previous cluster UUID. All cluster UUIDs evaluated are: {}", manifestsByClusterUUID.keySet());
             return Collections.emptyList();
         }
-        if (validClusterUUIDs.size() > 1) {
+        if (topLevelClusterUUIDs.size() > 1) {
+            logger.info("Top level cluster UUIDs: {}", topLevelClusterUUIDs);
             // If the valid cluster UUIDs are more that 1, it means there was some race condition where
             // more then 2 cluster manager nodes tried to become active cluster manager and published
             // 2 cluster UUIDs which followed the same previous UUID.
             final Map manifestsByClusterUUIDTrimmed = trimClusterUUIDs(
                 manifestsByClusterUUID,
-                validClusterUUIDs,
+                topLevelClusterUUIDs,
                 clusterName
             );
             if (manifestsByClusterUUID.size() == manifestsByClusterUUIDTrimmed.size()) {
@@ -908,14 +914,14 @@ private List createClusterChain(final Map validChain = new ArrayList<>();
-        String currentUUID = validClusterUUIDs.get(0);
+        String currentUUID = topLevelClusterUUIDs.get(0);
         while (currentUUID != null && !ClusterState.UNKNOWN_UUID.equals(currentUUID)) {
             validChain.add(currentUUID);
             // Getting the previous cluster UUID of a cluster UUID from the clusterUUID Graph
@@ -942,11 +948,7 @@ private Map trimClusterUUIDs(
             // Here we compare the manifest of current UUID to that of previous UUID
             // In case currentUUID's latest manifest is same as previous UUIDs latest manifest,
             // that means it was restored from previousUUID and no IndexMetadata update was performed on it.
-            if (ClusterState.UNKNOWN_UUID.equals(currentManifest.getPreviousClusterUUID())) {
-                if (currentManifest.getIndices().isEmpty()) {
-                    trimmedUUIDs.remove(clusterUUID);
-                }
-            } else {
+            if (!ClusterState.UNKNOWN_UUID.equals(currentManifest.getPreviousClusterUUID())) {
                 ClusterMetadataManifest previousManifest = trimmedUUIDs.get(currentManifest.getPreviousClusterUUID());
                 if (isMetadataEqual(currentManifest, previousManifest, clusterName)
                     && isGlobalMetadataEqual(currentManifest, previousManifest, clusterName)) {
@@ -985,8 +987,8 @@ private boolean isGlobalMetadataEqual(ClusterMetadataManifest first, ClusterMeta
         return Metadata.isGlobalResourcesMetadataEquals(firstGlobalMetadata, secondGlobalMetadata);
     }
 
-    private boolean isInvalidClusterUUID(ClusterMetadataManifest manifest) {
-        return !manifest.isClusterUUIDCommitted();
+    private boolean isValidClusterUUID(ClusterMetadataManifest manifest) {
+        return manifest.isClusterUUIDCommitted();
     }
 
     /**
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index ca88653f529f6..586618bd1ecff 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -911,7 +911,7 @@ public void testGetValidPreviousClusterUUIDWithMultipleChains() throws IOExcepti
             "cluster-uuid3",
             "cluster-uuid1"
         );
-        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, randomBoolean());
+        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, randomBoolean(), Collections.emptyMap());
 
         remoteClusterStateService.start();
         String previousClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster");
@@ -933,6 +933,23 @@ public void testGetValidPreviousClusterUUIDWithInvalidMultipleChains() throws IO
         assertThrows(IllegalStateException.class, () -> remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster"));
     }
 
+    public void testGetValidPreviousClusterUUIDWhenLastUUIDUncommitted() throws IOException {
+        Map clusterUUIDsPointers = Map.of(
+            "cluster-uuid1",
+            ClusterState.UNKNOWN_UUID,
+            "cluster-uuid2",
+            "cluster-uuid1",
+            "cluster-uuid3",
+            "cluster-uuid2"
+        );
+        Map clusterUUIDCommitted = Map.of("cluster-uuid1", true, "cluster-uuid2", true, "cluster-uuid3", false);
+        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, clusterUUIDCommitted);
+
+        remoteClusterStateService.start();
+        String previousClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster");
+        assertThat(previousClusterUUID, equalTo("cluster-uuid2"));
+    }
+
     public void testDeleteStaleClusterUUIDs() throws IOException {
         final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
         ClusterMetadataManifest clusterMetadataManifest = ClusterMetadataManifest.builder()
@@ -1128,11 +1145,21 @@ public void testGlobalMetadataUploadWaitTimeSetting() {
     }
 
     private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException {
-        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false);
+        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap());
     }
 
-    private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers, boolean differGlobalMetadata)
-        throws IOException {
+    private void mockObjectsForGettingPreviousClusterUUID(
+        Map clusterUUIDsPointers,
+        Map clusterUUIDCommitted
+    ) throws IOException {
+        mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, clusterUUIDCommitted);
+    }
+
+    private void mockObjectsForGettingPreviousClusterUUID(
+        Map clusterUUIDsPointers,
+        boolean differGlobalMetadata,
+        Map clusterUUIDCommitted
+    ) throws IOException {
         final BlobPath blobPath = mock(BlobPath.class);
         when((blobStoreRepository.basePath())).thenReturn(blobPath);
         when(blobPath.add(anyString())).thenReturn(blobPath);
@@ -1155,7 +1182,8 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste
             clusterUUIDsPointers.get("cluster-uuid1"),
             randomAlphaOfLength(10),
             uploadedIndexMetadataList1,
-            "test-metadata1"
+            "test-metadata1",
+            clusterUUIDCommitted.getOrDefault("cluster-uuid1", true)
         );
         Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexMetadata indexMetadata1 = IndexMetadata.builder("index1")
@@ -1184,7 +1212,8 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste
             clusterUUIDsPointers.get("cluster-uuid2"),
             randomAlphaOfLength(10),
             uploadedIndexMetadataList2,
-            "test-metadata2"
+            "test-metadata2",
+            clusterUUIDCommitted.getOrDefault("cluster-uuid2", true)
         );
         IndexMetadata indexMetadata3 = IndexMetadata.builder("index1")
             .settings(indexSettings)
@@ -1229,7 +1258,8 @@ private void mockObjectsForGettingPreviousClusterUUID(Map cluste
             clusterUUIDsPointers.get("cluster-uuid3"),
             randomAlphaOfLength(10),
             uploadedIndexMetadataList3,
-            "test-metadata3"
+            "test-metadata3",
+            clusterUUIDCommitted.getOrDefault("cluster-uuid3", true)
         );
         mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3);
         mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V1);
@@ -1257,7 +1287,8 @@ private ClusterMetadataManifest generateClusterMetadataManifest(
         String previousClusterUUID,
         String stateUUID,
         List uploadedIndexMetadata,
-        String globalMetadataFileName
+        String globalMetadataFileName,
+        Boolean isUUIDCommitted
     ) {
         return ClusterMetadataManifest.builder()
             .indices(uploadedIndexMetadata)
@@ -1269,7 +1300,7 @@ private ClusterMetadataManifest generateClusterMetadataManifest(
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID(previousClusterUUID)
             .committed(true)
-            .clusterUUIDCommitted(true)
+            .clusterUUIDCommitted(isUUIDCommitted)
             .globalMetadataFileName(globalMetadataFileName)
             .codecVersion(ClusterMetadataManifest.CODEC_V1)
             .build();

From 5ae93338e786173e39f5906ea1b2a5129f98ab66 Mon Sep 17 00:00:00 2001
From: Suraj Singh 
Date: Thu, 26 Oct 2023 07:42:23 -0700
Subject: [PATCH 242/388] Mute testSegRepSucceedsOnPreviousCopiedFiles and
 testNoFailuresOnFileReads unit tests (#10942)

Signed-off-by: Suraj Singh 
---
 .../java/org/opensearch/index/shard/RemoteIndexShardTests.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
index 2ce0bdc607189..20cec90d79e3e 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
@@ -362,6 +362,7 @@ public void testPrimaryRestart() throws Exception {
      * prevent FileAlreadyExistsException. It does so by only copying files in first round of segment replication without
      * committing locally so that in next round of segment replication those files are not considered for download again
      */
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10885")
     public void testSegRepSucceedsOnPreviousCopiedFiles() throws Exception {
         try (ReplicationGroup shards = createGroup(1, getIndexSettings(), new NRTReplicationEngineFactory())) {
             shards.startAll();
@@ -453,6 +454,7 @@ public void onReplicationFailure(
      * blocking update of reader. Once this is done, it corrupts one segment file and ensure that file is deleted in next
      * round of segment replication by ensuring doc count.
      */
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10885")
     public void testNoFailuresOnFileReads() throws Exception {
         try (ReplicationGroup shards = createGroup(1, getIndexSettings(), new NRTReplicationEngineFactory())) {
             shards.startAll();

From 0c9fc21ae78babd6820479ea940a4c986d82f10f Mon Sep 17 00:00:00 2001
From: Poojita Raj 
Date: Thu, 26 Oct 2023 16:11:19 -0700
Subject: [PATCH 243/388] Add log for failover time (#10952)

Signed-off-by: Poojita Raj 
---
 .../java/org/opensearch/index/shard/IndexShard.java    | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 5b6257084e440..352d4efc95269 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -187,6 +187,7 @@
 import org.opensearch.indices.recovery.RecoveryTarget;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher;
+import org.opensearch.indices.replication.common.ReplicationTimer;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
 import org.opensearch.search.suggest.completion.CompletionStats;
@@ -698,7 +699,16 @@ public void updateShardState(
                             if (indexSettings.isSegRepEnabled()) {
                                 // this Shard's engine was read only, we need to update its engine before restoring local history from xlog.
                                 assert newRouting.primary() && currentRouting.primary() == false;
+                                ReplicationTimer timer = new ReplicationTimer();
+                                timer.start();
+                                logger.debug(
+                                    "Resetting engine on promotion of shard [{}] to primary, startTime {}\n",
+                                    shardId,
+                                    timer.startTime()
+                                );
                                 resetEngineToGlobalCheckpoint();
+                                timer.stop();
+                                logger.info("Completed engine failover for shard [{}] in: {} ms", shardId, timer.time());
                                 // It is possible an engine can open with a SegmentInfos on a higher gen but the reader does not refresh to
                                 // trigger our refresh listener.
                                 // Force update the checkpoint post engine reset.

From e9affeab3494f4c2ed96a3efd647530b38a315fc Mon Sep 17 00:00:00 2001
From: rishavz_sagar 
Date: Fri, 27 Oct 2023 14:34:13 +0530
Subject: [PATCH 244/388] Fixing unreferenced file cleanup flaky tests (#10801)

Signed-off-by: RS146BIJAY 
---
 .../index/engine/InternalEngineTests.java     | 44 ++-----------------
 1 file changed, 4 insertions(+), 40 deletions(-)

diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
index 305c3a3acbf75..81d8bccb86c60 100644
--- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
@@ -40,7 +40,6 @@
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.appender.AbstractAppender;
 import org.apache.logging.log4j.core.filter.RegexFilter;
-import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.KeywordField;
 import org.apache.lucene.document.LongPoint;
@@ -3237,22 +3236,10 @@ public void testUnreferencedFileCleanUpOnSegmentMergeFailureWithCleanUpEnabled()
         MockDirectoryWrapper wrapper = newMockDirectory();
         final CountDownLatch cleanupCompleted = new CountDownLatch(1);
         MockDirectoryWrapper.Failure fail = new MockDirectoryWrapper.Failure() {
-            public boolean didFail1;
-            public boolean didFail2;
-
             @Override
             public void eval(MockDirectoryWrapper dir) throws IOException {
-                if (!doFail) {
-                    return;
-                }
-
-                // Fail segment merge with diskfull during merging terms.
-                if (callStackContainsAnyOf("mergeTerms") && !didFail1) {
-                    didFail1 = true;
-                    throw new IOException("No space left on device");
-                }
-                if (callStackContains(LiveDocsFormat.class, "writeLiveDocs") && !didFail2) {
-                    didFail2 = true;
+                // Fail segment merge with diskfull during merging terms
+                if (callStackContainsAnyOf("mergeTerms")) {
                     throw new IOException("No space left on device");
                 }
             }
@@ -3325,7 +3312,6 @@ public void onFailedEngine(String reason, Exception e) {
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(2));
 
-            fail.setDoFail();
             // IndexWriter can throw either IOException or IllegalStateException depending on whether tragedy is set or not.
             expectThrowsAnyOf(
                 Arrays.asList(IOException.class, IllegalStateException.class),
@@ -3345,20 +3331,10 @@ public void testUnreferencedFileCleanUpOnSegmentMergeFailureWithCleanUpDisabled(
         MockDirectoryWrapper wrapper = newMockDirectory();
         final CountDownLatch cleanupCompleted = new CountDownLatch(1);
         MockDirectoryWrapper.Failure fail = new MockDirectoryWrapper.Failure() {
-            public boolean didFail1;
-            public boolean didFail2;
 
             @Override
             public void eval(MockDirectoryWrapper dir) throws IOException {
-                if (!doFail) {
-                    return;
-                }
-                if (callStackContainsAnyOf("mergeTerms") && !didFail1) {
-                    didFail1 = true;
-                    throw new IOException("No space left on device");
-                }
-                if (callStackContains(LiveDocsFormat.class, "writeLiveDocs") && !didFail2) {
-                    didFail2 = true;
+                if (callStackContainsAnyOf("mergeTerms")) {
                     throw new IOException("No space left on device");
                 }
             }
@@ -3439,7 +3415,6 @@ public void onFailedEngine(String reason, Exception e) {
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(2));
 
-            fail.setDoFail();
             // IndexWriter can throw either IOException or IllegalStateException depending on whether tragedy is set or not.
             expectThrowsAnyOf(
                 Arrays.asList(IOException.class, IllegalStateException.class),
@@ -3459,20 +3434,10 @@ public void testUnreferencedFileCleanUpFailsOnSegmentMergeFailureWhenDirectoryCl
         MockDirectoryWrapper wrapper = newMockDirectory();
         final CountDownLatch cleanupCompleted = new CountDownLatch(1);
         MockDirectoryWrapper.Failure fail = new MockDirectoryWrapper.Failure() {
-            public boolean didFail1;
-            public boolean didFail2;
 
             @Override
             public void eval(MockDirectoryWrapper dir) throws IOException {
-                if (!doFail) {
-                    return;
-                }
-                if (callStackContainsAnyOf("mergeTerms") && !didFail1) {
-                    didFail1 = true;
-                    throw new IOException("No space left on device");
-                }
-                if (callStackContains(LiveDocsFormat.class, "writeLiveDocs") && !didFail2) {
-                    didFail2 = true;
+                if (callStackContainsAnyOf("mergeTerms")) {
                     throw new IOException("No space left on device");
                 }
             }
@@ -3537,7 +3502,6 @@ public void onFailedEngine(String reason, Exception e) {
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(2));
 
-            fail.setDoFail();
             // Close the store so that unreferenced file cleanup will fail.
             store.close();
 

From be65f543d125965d699e493933851d0cfec7e530 Mon Sep 17 00:00:00 2001
From: Ketan Verma <9292653+ketanv3@users.noreply.github.com>
Date: Fri, 27 Oct 2023 18:02:29 +0530
Subject: [PATCH 245/388] Remove deprecated classes for Rounding (#10956)

Signed-off-by: Ketan Verma 
---
 CHANGELOG.md                                  |   1 +
 .../benchmark/time/RoundingBenchmark.java     | 180 ----
 .../common/rounding/DateTimeUnit.java         |  99 ---
 .../opensearch/common/rounding/Rounding.java  | 459 ----------
 .../common/rounding/package-info.java         |  10 -
 .../org/opensearch/common/RoundingTests.java  |   3 +-
 .../common/rounding/DateTimeUnitTests.java    |  75 --
 .../common/rounding/RoundingDuelTests.java    |  70 --
 .../rounding/TimeZoneRoundingTests.java       | 822 ------------------
 9 files changed, 2 insertions(+), 1717 deletions(-)
 delete mode 100644 benchmarks/src/main/java/org/opensearch/benchmark/time/RoundingBenchmark.java
 delete mode 100644 server/src/main/java/org/opensearch/common/rounding/DateTimeUnit.java
 delete mode 100644 server/src/main/java/org/opensearch/common/rounding/Rounding.java
 delete mode 100644 server/src/main/java/org/opensearch/common/rounding/package-info.java
 delete mode 100644 server/src/test/java/org/opensearch/common/rounding/DateTimeUnitTests.java
 delete mode 100644 server/src/test/java/org/opensearch/common/rounding/RoundingDuelTests.java
 delete mode 100644 server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b40878066960a..234b08398f9ef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -124,6 +124,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Deprecated
 
 ### Removed
+- Remove deprecated classes for Rounding ([#10956](https://github.com/opensearch-project/OpenSearch/issues/10956))
 
 ### Fixed
 - Fix failure in dissect ingest processor parsing empty brackets ([#9225](https://github.com/opensearch-project/OpenSearch/pull/9255))
diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/time/RoundingBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/time/RoundingBenchmark.java
deleted file mode 100644
index cdbcbfc163191..0000000000000
--- a/benchmarks/src/main/java/org/opensearch/benchmark/time/RoundingBenchmark.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.benchmark.time;
-
-import org.opensearch.common.Rounding;
-import org.opensearch.common.rounding.DateTimeUnit;
-import org.opensearch.common.time.DateUtils;
-import org.opensearch.common.unit.TimeValue;
-import org.joda.time.DateTimeZone;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.Warmup;
-
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.util.concurrent.TimeUnit;
-
-import static org.opensearch.common.Rounding.DateTimeUnit.DAY_OF_MONTH;
-import static org.opensearch.common.Rounding.DateTimeUnit.MONTH_OF_YEAR;
-import static org.opensearch.common.Rounding.DateTimeUnit.QUARTER_OF_YEAR;
-import static org.opensearch.common.Rounding.DateTimeUnit.YEAR_OF_CENTURY;
-
-@Fork(3)
-@Warmup(iterations = 10)
-@Measurement(iterations = 10)
-@BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.NANOSECONDS)
-@State(Scope.Benchmark)
-@SuppressWarnings("unused") // invoked by benchmarking framework
-public class RoundingBenchmark {
-
-    private final ZoneId zoneId = ZoneId.of("Europe/Amsterdam");
-    private final DateTimeZone timeZone = DateUtils.zoneIdToDateTimeZone(zoneId);
-
-    private long timestamp = 1548879021354L;
-
-    private final org.opensearch.common.rounding.Rounding jodaRounding = org.opensearch.common.rounding.Rounding.builder(
-        DateTimeUnit.HOUR_OF_DAY
-    ).timeZone(timeZone).build();
-    private final Rounding javaRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).timeZone(zoneId).build();
-
-    @Benchmark
-    public long timeRoundingDateTimeUnitJoda() {
-        return jodaRounding.round(timestamp);
-    }
-
-    @Benchmark
-    public long timeRoundingDateTimeUnitJava() {
-        return javaRounding.round(timestamp);
-    }
-
-    private final org.opensearch.common.rounding.Rounding jodaDayOfMonthRounding = org.opensearch.common.rounding.Rounding.builder(
-        DateTimeUnit.DAY_OF_MONTH
-    ).timeZone(timeZone).build();
-    private final Rounding javaDayOfMonthRounding = Rounding.builder(DAY_OF_MONTH).timeZone(zoneId).build();
-
-    @Benchmark
-    public long timeRoundingDateTimeUnitDayOfMonthJoda() {
-        return jodaDayOfMonthRounding.round(timestamp);
-    }
-
-    @Benchmark
-    public long timeRoundingDateTimeUnitDayOfMonthJava() {
-        return javaDayOfMonthRounding.round(timestamp);
-    }
-
-    private final org.opensearch.common.rounding.Rounding timeIntervalRoundingJoda = org.opensearch.common.rounding.Rounding.builder(
-        TimeValue.timeValueMinutes(60)
-    ).timeZone(timeZone).build();
-    private final Rounding timeIntervalRoundingJava = Rounding.builder(TimeValue.timeValueMinutes(60)).timeZone(zoneId).build();
-
-    @Benchmark
-    public long timeIntervalRoundingJava() {
-        return timeIntervalRoundingJava.round(timestamp);
-    }
-
-    @Benchmark
-    public long timeIntervalRoundingJoda() {
-        return timeIntervalRoundingJoda.round(timestamp);
-    }
-
-    private final org.opensearch.common.rounding.Rounding timeUnitRoundingUtcDayOfMonthJoda = org.opensearch.common.rounding.Rounding
-        .builder(DateTimeUnit.DAY_OF_MONTH)
-        .timeZone(DateTimeZone.UTC)
-        .build();
-    private final Rounding timeUnitRoundingUtcDayOfMonthJava = Rounding.builder(DAY_OF_MONTH).timeZone(ZoneOffset.UTC).build();
-
-    @Benchmark
-    public long timeUnitRoundingUtcDayOfMonthJava() {
-        return timeUnitRoundingUtcDayOfMonthJava.round(timestamp);
-    }
-
-    @Benchmark
-    public long timeUnitRoundingUtcDayOfMonthJoda() {
-        return timeUnitRoundingUtcDayOfMonthJoda.round(timestamp);
-    }
-
-    private final org.opensearch.common.rounding.Rounding timeUnitRoundingUtcQuarterOfYearJoda = org.opensearch.common.rounding.Rounding
-        .builder(DateTimeUnit.QUARTER)
-        .timeZone(DateTimeZone.UTC)
-        .build();
-    private final Rounding timeUnitRoundingUtcQuarterOfYearJava = Rounding.builder(QUARTER_OF_YEAR).timeZone(ZoneOffset.UTC).build();
-
-    @Benchmark
-    public long timeUnitRoundingUtcQuarterOfYearJava() {
-        return timeUnitRoundingUtcQuarterOfYearJava.round(timestamp);
-    }
-
-    @Benchmark
-    public long timeUnitRoundingUtcQuarterOfYearJoda() {
-        return timeUnitRoundingUtcQuarterOfYearJoda.round(timestamp);
-    }
-
-    private final org.opensearch.common.rounding.Rounding timeUnitRoundingUtcMonthOfYearJoda = org.opensearch.common.rounding.Rounding
-        .builder(DateTimeUnit.MONTH_OF_YEAR)
-        .timeZone(DateTimeZone.UTC)
-        .build();
-    private final Rounding timeUnitRoundingUtcMonthOfYearJava = Rounding.builder(MONTH_OF_YEAR).timeZone(ZoneOffset.UTC).build();
-
-    @Benchmark
-    public long timeUnitRoundingUtcMonthOfYearJava() {
-        return timeUnitRoundingUtcMonthOfYearJava.round(timestamp);
-    }
-
-    @Benchmark
-    public long timeUnitRoundingUtcMonthOfYearJoda() {
-        return timeUnitRoundingUtcMonthOfYearJoda.round(timestamp);
-    }
-
-    private final org.opensearch.common.rounding.Rounding timeUnitRoundingUtcYearOfCenturyJoda = org.opensearch.common.rounding.Rounding
-        .builder(DateTimeUnit.YEAR_OF_CENTURY)
-        .timeZone(DateTimeZone.UTC)
-        .build();
-    private final Rounding timeUnitRoundingUtcYearOfCenturyJava = Rounding.builder(YEAR_OF_CENTURY).timeZone(ZoneOffset.UTC).build();
-
-    @Benchmark
-    public long timeUnitRoundingUtcYearOfCenturyJava() {
-        return timeUnitRoundingUtcYearOfCenturyJava.round(timestamp);
-    }
-
-    @Benchmark
-    public long timeUnitRoundingUtcYearOfCenturyJoda() {
-        return timeUnitRoundingUtcYearOfCenturyJoda.round(timestamp);
-    }
-}
diff --git a/server/src/main/java/org/opensearch/common/rounding/DateTimeUnit.java b/server/src/main/java/org/opensearch/common/rounding/DateTimeUnit.java
deleted file mode 100644
index 47e182b3caf84..0000000000000
--- a/server/src/main/java/org/opensearch/common/rounding/DateTimeUnit.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.common.rounding;
-
-import org.opensearch.OpenSearchException;
-import org.opensearch.common.joda.Joda;
-import org.joda.time.DateTimeField;
-import org.joda.time.DateTimeZone;
-import org.joda.time.chrono.ISOChronology;
-
-import java.util.function.Function;
-
-/**
- * Main date time unit class.
- *
- * @opensearch.internal
- */
-public enum DateTimeUnit {
-
-    WEEK_OF_WEEKYEAR((byte) 1, tz -> ISOChronology.getInstance(tz).weekOfWeekyear()),
-    YEAR_OF_CENTURY((byte) 2, tz -> ISOChronology.getInstance(tz).yearOfCentury()),
-    QUARTER((byte) 3, tz -> Joda.QuarterOfYear.getField(ISOChronology.getInstance(tz))),
-    MONTH_OF_YEAR((byte) 4, tz -> ISOChronology.getInstance(tz).monthOfYear()),
-    DAY_OF_MONTH((byte) 5, tz -> ISOChronology.getInstance(tz).dayOfMonth()),
-    HOUR_OF_DAY((byte) 6, tz -> ISOChronology.getInstance(tz).hourOfDay()),
-    MINUTES_OF_HOUR((byte) 7, tz -> ISOChronology.getInstance(tz).minuteOfHour()),
-    SECOND_OF_MINUTE((byte) 8, tz -> ISOChronology.getInstance(tz).secondOfMinute());
-
-    private final byte id;
-    private final Function fieldFunction;
-
-    DateTimeUnit(byte id, Function fieldFunction) {
-        this.id = id;
-        this.fieldFunction = fieldFunction;
-    }
-
-    public byte id() {
-        return id;
-    }
-
-    /**
-     * @return the {@link DateTimeField} for the provided {@link DateTimeZone} for this time unit
-     */
-    public DateTimeField field(DateTimeZone tz) {
-        return fieldFunction.apply(tz);
-    }
-
-    public static DateTimeUnit resolve(byte id) {
-        switch (id) {
-            case 1:
-                return WEEK_OF_WEEKYEAR;
-            case 2:
-                return YEAR_OF_CENTURY;
-            case 3:
-                return QUARTER;
-            case 4:
-                return MONTH_OF_YEAR;
-            case 5:
-                return DAY_OF_MONTH;
-            case 6:
-                return HOUR_OF_DAY;
-            case 7:
-                return MINUTES_OF_HOUR;
-            case 8:
-                return SECOND_OF_MINUTE;
-            default:
-                throw new OpenSearchException("Unknown date time unit id [" + id + "]");
-        }
-    }
-}
diff --git a/server/src/main/java/org/opensearch/common/rounding/Rounding.java b/server/src/main/java/org/opensearch/common/rounding/Rounding.java
deleted file mode 100644
index 41e808b64f7d9..0000000000000
--- a/server/src/main/java/org/opensearch/common/rounding/Rounding.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.common.rounding;
-
-import org.opensearch.OpenSearchException;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.common.io.stream.Writeable;
-import org.joda.time.DateTimeField;
-import org.joda.time.DateTimeZone;
-import org.joda.time.IllegalInstantException;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * A strategy for rounding long values.
- * 
                              
- * Use the java based Rounding class where applicable
- *
- * @opensearch.internal
- */
-@Deprecated
-public abstract class Rounding implements Writeable {
-
-    public abstract byte id();
-
-    /**
-     * Rounds the given value.
-     */
-    public abstract long round(long value);
-
-    /**
-     * Given the rounded value (which was potentially generated by {@link #round(long)}, returns the next rounding value. For example, with
-     * interval based rounding, if the interval is 3, {@code nextRoundValue(6) = 9 }.
-     *
-     * @param value The current rounding value
-     * @return      The next rounding value;
-     */
-    public abstract long nextRoundingValue(long value);
-
-    @Override
-    public abstract boolean equals(Object obj);
-
-    @Override
-    public abstract int hashCode();
-
-    public static Builder builder(DateTimeUnit unit) {
-        return new Builder(unit);
-    }
-
-    public static Builder builder(TimeValue interval) {
-        return new Builder(interval);
-    }
-
-    /**
-     * Builder for rounding
-     *
-     * @opensearch.internal
-     */
-    public static class Builder {
-
-        private final DateTimeUnit unit;
-        private final long interval;
-
-        private DateTimeZone timeZone = DateTimeZone.UTC;
-
-        public Builder(DateTimeUnit unit) {
-            this.unit = unit;
-            this.interval = -1;
-        }
-
-        public Builder(TimeValue interval) {
-            this.unit = null;
-            if (interval.millis() < 1) throw new IllegalArgumentException("Zero or negative time interval not supported");
-            this.interval = interval.millis();
-        }
-
-        public Builder timeZone(DateTimeZone timeZone) {
-            if (timeZone == null) {
-                throw new IllegalArgumentException("Setting null as timezone is not supported");
-            }
-            this.timeZone = timeZone;
-            return this;
-        }
-
-        public Rounding build() {
-            Rounding timeZoneRounding;
-            if (unit != null) {
-                timeZoneRounding = new TimeUnitRounding(unit, timeZone);
-            } else {
-                timeZoneRounding = new TimeIntervalRounding(interval, timeZone);
-            }
-            return timeZoneRounding;
-        }
-    }
-
-    /**
-     * Rounding time units
-     *
-     * @opensearch.internal
-     */
-    static class TimeUnitRounding extends Rounding {
-
-        static final byte ID = 1;
-
-        private final DateTimeUnit unit;
-        private final DateTimeField field;
-        private final DateTimeZone timeZone;
-        private final boolean unitRoundsToMidnight;
-
-        TimeUnitRounding(DateTimeUnit unit, DateTimeZone timeZone) {
-            this.unit = unit;
-            this.field = unit.field(timeZone);
-            unitRoundsToMidnight = this.field.getDurationField().getUnitMillis() > 60L * 60L * 1000L;
-            this.timeZone = timeZone;
-        }
-
-        TimeUnitRounding(StreamInput in) throws IOException {
-            unit = DateTimeUnit.resolve(in.readByte());
-            timeZone = DateTimeZone.forID(in.readString());
-            field = unit.field(timeZone);
-            unitRoundsToMidnight = field.getDurationField().getUnitMillis() > 60L * 60L * 1000L;
-        }
-
-        @Override
-        public byte id() {
-            return ID;
-        }
-
-        /**
-         * @return The latest timestamp T which is strictly before utcMillis
-         * and such that timeZone.getOffset(T) != timeZone.getOffset(utcMillis).
-         * If there is no such T, returns Long.MAX_VALUE.
-         */
-        private long previousTransition(long utcMillis) {
-            final int offsetAtInputTime = timeZone.getOffset(utcMillis);
-            do {
-                // Some timezones have transitions that do not change the offset, so we have to
-                // repeatedly call previousTransition until a nontrivial transition is found.
-
-                long previousTransition = timeZone.previousTransition(utcMillis);
-                if (previousTransition == utcMillis) {
-                    // There are no earlier transitions
-                    return Long.MAX_VALUE;
-                }
-                assert previousTransition < utcMillis; // Progress was made
-                utcMillis = previousTransition;
-            } while (timeZone.getOffset(utcMillis) == offsetAtInputTime);
-
-            return utcMillis;
-        }
-
-        @Override
-        public long round(long utcMillis) {
-
-            // field.roundFloor() works as long as the offset doesn't change. It is worth getting this case out of the way first, as
-            // the calculations for fixing things near to offset changes are a little expensive and are unnecessary in the common case
-            // of working in UTC.
-            if (timeZone.isFixed()) {
-                return field.roundFloor(utcMillis);
-            }
-
-            // When rounding to hours we consider any local time of the form 'xx:00:00' as rounded, even though this gives duplicate
-            // bucket names for the times when the clocks go back. Shorter units behave similarly. However, longer units round down to
-            // midnight, and on the days where there are two midnights we would rather pick the earlier one, so that buckets are
-            // uniquely identified by the date.
-            if (unitRoundsToMidnight) {
-                final long anyLocalStartOfDay = field.roundFloor(utcMillis);
-                // `anyLocalStartOfDay` is _supposed_ to be the Unix timestamp for the start of the day in question in the current time
-                // zone. Mostly this just means "midnight", which is fine, and on days with no local midnight it's the first time that
-                // does occur on that day which is also ok. However, on days with >1 local midnight this is _one_ of the midnights, but
-                // may not be the first. Check whether this is happening, and fix it if so.
-
-                final long previousTransition = previousTransition(anyLocalStartOfDay);
-
-                if (previousTransition == Long.MAX_VALUE) {
-                    // No previous transitions, so there can't be another earlier local midnight.
-                    return anyLocalStartOfDay;
-                }
-
-                final long currentOffset = timeZone.getOffset(anyLocalStartOfDay);
-                final long previousOffset = timeZone.getOffset(previousTransition);
-                assert currentOffset != previousOffset;
-
-                // NB we only assume interference from one previous transition. It's theoretically possible to have two transitions in
-                // quick succession, both of which have a midnight in them, but this doesn't appear to happen in the TZDB so (a) it's
-                // pointless to implement and (b) it won't be tested. I recognise that this comment is tempting fate and will likely
-                // cause this very situation to occur in the near future, and eagerly look forward to fixing this using a loop over
-                // previous transitions when it happens.
-
-                final long alsoLocalStartOfDay = anyLocalStartOfDay + currentOffset - previousOffset;
-                // `alsoLocalStartOfDay` is the Unix timestamp for the start of the day in question if the previous offset were in
-                // effect.
-
-                if (alsoLocalStartOfDay <= previousTransition) {
-                    // Therefore the previous offset _is_ in effect at `alsoLocalStartOfDay`, and it's earlier than anyLocalStartOfDay,
-                    // so this is the answer to use.
-                    return alsoLocalStartOfDay;
-                } else {
-                    // The previous offset is not in effect at `alsoLocalStartOfDay`, so the current offset must be.
-                    return anyLocalStartOfDay;
-                }
-
-            } else {
-                do {
-                    long rounded = field.roundFloor(utcMillis);
-
-                    // field.roundFloor() mostly works as long as the offset hasn't changed in [rounded, utcMillis], so look at where
-                    // the offset most recently changed.
-
-                    final long previousTransition = previousTransition(utcMillis);
-
-                    if (previousTransition == Long.MAX_VALUE || previousTransition < rounded) {
-                        // The offset did not change in [rounded, utcMillis], so roundFloor() worked as expected.
-                        return rounded;
-                    }
-
-                    // The offset _did_ change in [rounded, utcMillis]. Put differently, this means that none of the times in
-                    // [previousTransition+1, utcMillis] were rounded, so the rounded time must be <= previousTransition. This means
-                    // it's sufficient to try and round previousTransition down.
-                    assert previousTransition < utcMillis;
-                    utcMillis = previousTransition;
-                } while (true);
-            }
-        }
-
-        @Override
-        public long nextRoundingValue(long utcMillis) {
-            long floor = round(utcMillis);
-            // add one unit and round to get to next rounded value
-            long next = round(field.add(floor, 1));
-            if (next == floor) {
-                // in rare case we need to add more than one unit
-                next = round(field.add(floor, 2));
-            }
-            return next;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeByte(unit.id());
-            out.writeString(timeZone.getID());
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(unit, timeZone);
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (obj == null) {
-                return false;
-            }
-            if (getClass() != obj.getClass()) {
-                return false;
-            }
-            TimeUnitRounding other = (TimeUnitRounding) obj;
-            return Objects.equals(unit, other.unit) && Objects.equals(timeZone, other.timeZone);
-        }
-
-        @Override
-        public String toString() {
-            return "[" + timeZone + "][" + unit + "]";
-        }
-    }
-
-    /**
-     * Rounding time intervals
-     *
-     * @opensearch.internal
-     */
-    static class TimeIntervalRounding extends Rounding {
-
-        static final byte ID = 2;
-
-        private final long interval;
-        private final DateTimeZone timeZone;
-
-        TimeIntervalRounding(long interval, DateTimeZone timeZone) {
-            if (interval < 1) throw new IllegalArgumentException("Zero or negative time interval not supported");
-            this.interval = interval;
-            this.timeZone = timeZone;
-        }
-
-        TimeIntervalRounding(StreamInput in) throws IOException {
-            interval = in.readVLong();
-            timeZone = DateTimeZone.forID(in.readString());
-        }
-
-        @Override
-        public byte id() {
-            return ID;
-        }
-
-        @Override
-        public long round(long utcMillis) {
-            long timeLocal = timeZone.convertUTCToLocal(utcMillis);
-            long rounded = roundKey(timeLocal, interval) * interval;
-            long roundedUTC;
-            if (isInDSTGap(rounded) == false) {
-                roundedUTC = timeZone.convertLocalToUTC(rounded, true, utcMillis);
-                // check if we crossed DST transition, in this case we want the
-                // last rounded value before the transition
-                long transition = timeZone.previousTransition(utcMillis);
-                if (transition != utcMillis && transition > roundedUTC) {
-                    roundedUTC = round(transition - 1);
-                }
-            } else {
-                /*
-                 * Edge case where the rounded local time is illegal and landed
-                 * in a DST gap. In this case, we choose 1ms tick after the
-                 * transition date. We don't want the transition date itself
-                 * because those dates, when rounded themselves, fall into the
-                 * previous interval. This would violate the invariant that the
-                 * rounding operation should be idempotent.
-                 */
-                roundedUTC = timeZone.previousTransition(utcMillis) + 1;
-            }
-            return roundedUTC;
-        }
-
-        private static long roundKey(long value, long interval) {
-            if (value < 0) {
-                return (value - interval + 1) / interval;
-            } else {
-                return value / interval;
-            }
-        }
-
-        /**
-         * Determine whether the local instant is a valid instant in the given
-         * time zone. The logic for this is taken from
-         * {@link DateTimeZone#convertLocalToUTC(long, boolean)} for the
-         * `strict` mode case, but instead of throwing an
-         * {@link IllegalInstantException}, which is costly, we want to return a
-         * flag indicating that the value is illegal in that time zone.
-         */
-        private boolean isInDSTGap(long instantLocal) {
-            if (timeZone.isFixed()) {
-                return false;
-            }
-            // get the offset at instantLocal (first estimate)
-            int offsetLocal = timeZone.getOffset(instantLocal);
-            // adjust instantLocal using the estimate and recalc the offset
-            int offset = timeZone.getOffset(instantLocal - offsetLocal);
-            // if the offsets differ, we must be near a DST boundary
-            if (offsetLocal != offset) {
-                // determine if we are in the DST gap
-                long nextLocal = timeZone.nextTransition(instantLocal - offsetLocal);
-                if (nextLocal == (instantLocal - offsetLocal)) {
-                    nextLocal = Long.MAX_VALUE;
-                }
-                long nextAdjusted = timeZone.nextTransition(instantLocal - offset);
-                if (nextAdjusted == (instantLocal - offset)) {
-                    nextAdjusted = Long.MAX_VALUE;
-                }
-                if (nextLocal != nextAdjusted) {
-                    // we are in the DST gap
-                    return true;
-                }
-            }
-            return false;
-        }
-
-        @Override
-        public long nextRoundingValue(long time) {
-            long timeLocal = time;
-            timeLocal = timeZone.convertUTCToLocal(time);
-            long next = timeLocal + interval;
-            return timeZone.convertLocalToUTC(next, false);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVLong(interval);
-            out.writeString(timeZone.getID());
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(interval, timeZone);
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (obj == null) {
-                return false;
-            }
-            if (getClass() != obj.getClass()) {
-                return false;
-            }
-            TimeIntervalRounding other = (TimeIntervalRounding) obj;
-            return Objects.equals(interval, other.interval) && Objects.equals(timeZone, other.timeZone);
-        }
-    }
-
-    /**
-     * Rounding streams
-     *
-     * @opensearch.internal
-     */
-    public static class Streams {
-
-        public static void write(Rounding rounding, StreamOutput out) throws IOException {
-            out.writeByte(rounding.id());
-            rounding.writeTo(out);
-        }
-
-        public static Rounding read(StreamInput in) throws IOException {
-            Rounding rounding;
-            byte id = in.readByte();
-            switch (id) {
-                case TimeUnitRounding.ID:
-                    rounding = new TimeUnitRounding(in);
-                    break;
-                case TimeIntervalRounding.ID:
-                    rounding = new TimeIntervalRounding(in);
-                    break;
-                default:
-                    throw new OpenSearchException("unknown rounding id [" + id + "]");
-            }
-            return rounding;
-        }
-
-    }
-
-}
diff --git a/server/src/main/java/org/opensearch/common/rounding/package-info.java b/server/src/main/java/org/opensearch/common/rounding/package-info.java
deleted file mode 100644
index 5fa3e39c6a786..0000000000000
--- a/server/src/main/java/org/opensearch/common/rounding/package-info.java
+++ /dev/null
@@ -1,10 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/** Base DateTime rounding package. */
-package org.opensearch.common.rounding;
diff --git a/server/src/test/java/org/opensearch/common/RoundingTests.java b/server/src/test/java/org/opensearch/common/RoundingTests.java
index 1a499bac3e2e8..cc71ee08abcca 100644
--- a/server/src/test/java/org/opensearch/common/RoundingTests.java
+++ b/server/src/test/java/org/opensearch/common/RoundingTests.java
@@ -33,7 +33,6 @@
 package org.opensearch.common;
 
 import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.rounding.DateTimeUnit;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.time.DateFormatters;
 import org.opensearch.common.unit.TimeValue;
@@ -236,7 +235,7 @@ public void testOffsetRounding() {
 
     /**
      * Randomized test on TimeUnitRounding. Test uses random
-     * {@link DateTimeUnit} and {@link ZoneId} and often (50% of the time)
+     * {@link org.opensearch.common.Rounding.DateTimeUnit} and {@link ZoneId} and often (50% of the time)
      * chooses test dates that are exactly on or close to offset changes (e.g.
      * DST) in the chosen time zone.
      * 
                              
diff --git a/server/src/test/java/org/opensearch/common/rounding/DateTimeUnitTests.java b/server/src/test/java/org/opensearch/common/rounding/DateTimeUnitTests.java
deleted file mode 100644
index 7b87e136c5f38..0000000000000
--- a/server/src/test/java/org/opensearch/common/rounding/DateTimeUnitTests.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.common.rounding;
-
-import org.opensearch.test.OpenSearchTestCase;
-
-import static org.opensearch.common.rounding.DateTimeUnit.DAY_OF_MONTH;
-import static org.opensearch.common.rounding.DateTimeUnit.HOUR_OF_DAY;
-import static org.opensearch.common.rounding.DateTimeUnit.MINUTES_OF_HOUR;
-import static org.opensearch.common.rounding.DateTimeUnit.MONTH_OF_YEAR;
-import static org.opensearch.common.rounding.DateTimeUnit.QUARTER;
-import static org.opensearch.common.rounding.DateTimeUnit.SECOND_OF_MINUTE;
-import static org.opensearch.common.rounding.DateTimeUnit.WEEK_OF_WEEKYEAR;
-import static org.opensearch.common.rounding.DateTimeUnit.YEAR_OF_CENTURY;
-
-public class DateTimeUnitTests extends OpenSearchTestCase {
-
-    /**
-     * test that we don't accidentally change enum ids
-     */
-    public void testEnumIds() {
-        assertEquals(1, WEEK_OF_WEEKYEAR.id());
-        assertEquals(WEEK_OF_WEEKYEAR, DateTimeUnit.resolve((byte) 1));
-
-        assertEquals(2, YEAR_OF_CENTURY.id());
-        assertEquals(YEAR_OF_CENTURY, DateTimeUnit.resolve((byte) 2));
-
-        assertEquals(3, QUARTER.id());
-        assertEquals(QUARTER, DateTimeUnit.resolve((byte) 3));
-
-        assertEquals(4, MONTH_OF_YEAR.id());
-        assertEquals(MONTH_OF_YEAR, DateTimeUnit.resolve((byte) 4));
-
-        assertEquals(5, DAY_OF_MONTH.id());
-        assertEquals(DAY_OF_MONTH, DateTimeUnit.resolve((byte) 5));
-
-        assertEquals(6, HOUR_OF_DAY.id());
-        assertEquals(HOUR_OF_DAY, DateTimeUnit.resolve((byte) 6));
-
-        assertEquals(7, MINUTES_OF_HOUR.id());
-        assertEquals(MINUTES_OF_HOUR, DateTimeUnit.resolve((byte) 7));
-
-        assertEquals(8, SECOND_OF_MINUTE.id());
-        assertEquals(SECOND_OF_MINUTE, DateTimeUnit.resolve((byte) 8));
-    }
-}
diff --git a/server/src/test/java/org/opensearch/common/rounding/RoundingDuelTests.java b/server/src/test/java/org/opensearch/common/rounding/RoundingDuelTests.java
deleted file mode 100644
index 3088067cd1f84..0000000000000
--- a/server/src/test/java/org/opensearch/common/rounding/RoundingDuelTests.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.common.rounding;
-
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.test.OpenSearchTestCase;
-import org.joda.time.DateTimeZone;
-
-import java.time.ZoneOffset;
-
-import static org.hamcrest.Matchers.is;
-
-public class RoundingDuelTests extends OpenSearchTestCase {
-
-    // dont include nano/micro seconds as rounding would become zero then and throw an exception
-    private static final String[] ALLOWED_TIME_SUFFIXES = new String[] { "d", "h", "ms", "s", "m" };
-
-    public void testDuellingImplementations() {
-        org.opensearch.common.Rounding.DateTimeUnit randomDateTimeUnit = randomFrom(org.opensearch.common.Rounding.DateTimeUnit.values());
-        org.opensearch.common.Rounding.Prepared rounding;
-        Rounding roundingJoda;
-
-        if (randomBoolean()) {
-            rounding = org.opensearch.common.Rounding.builder(randomDateTimeUnit).timeZone(ZoneOffset.UTC).build().prepareForUnknown();
-            DateTimeUnit dateTimeUnit = DateTimeUnit.resolve(randomDateTimeUnit.getId());
-            roundingJoda = Rounding.builder(dateTimeUnit).timeZone(DateTimeZone.UTC).build();
-        } else {
-            TimeValue interval = timeValue();
-            rounding = org.opensearch.common.Rounding.builder(interval).timeZone(ZoneOffset.UTC).build().prepareForUnknown();
-            roundingJoda = Rounding.builder(interval).timeZone(DateTimeZone.UTC).build();
-        }
-
-        long roundValue = randomLong();
-        assertThat(roundingJoda.round(roundValue), is(rounding.round(roundValue)));
-    }
-
-    static TimeValue timeValue() {
-        return TimeValue.parseTimeValue(randomIntBetween(1, 1000) + randomFrom(ALLOWED_TIME_SUFFIXES), "settingName");
-    }
-}
diff --git a/server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java b/server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java
deleted file mode 100644
index d1b3adcd55f0c..0000000000000
--- a/server/src/test/java/org/opensearch/common/rounding/TimeZoneRoundingTests.java
+++ /dev/null
@@ -1,822 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.common.rounding;
-
-import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.rounding.Rounding.TimeIntervalRounding;
-import org.opensearch.common.rounding.Rounding.TimeUnitRounding;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.test.OpenSearchTestCase;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeConstants;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
-import org.hamcrest.TypeSafeMatcher;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-import static org.hamcrest.Matchers.startsWith;
-
-public class TimeZoneRoundingTests extends OpenSearchTestCase {
-
-    public void testUTCTimeUnitRounding() {
-        Rounding tzRounding = Rounding.builder(DateTimeUnit.MONTH_OF_YEAR).build();
-        DateTimeZone tz = DateTimeZone.UTC;
-        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-01T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-01T00:00:00.000Z")), isDate(time("2009-03-01T00:00:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.WEEK_OF_WEEKYEAR).build();
-        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-09T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-01-16T00:00:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.QUARTER).build();
-        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-01T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-04-01T00:00:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.HOUR_OF_DAY).build();
-        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-10T01:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-01-09T01:00:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.DAY_OF_MONTH).build();
-        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-10T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-01-10T00:00:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.YEAR_OF_CENTURY).build();
-        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-01T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2013-01-01T00:00:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.MINUTES_OF_HOUR).build();
-        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-10T01:01:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-01-09T00:01:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.SECOND_OF_MINUTE).build();
-        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-10T01:01:01.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-01-09T00:00:01.000Z"), tz));
-    }
-
-    public void testUTCIntervalRounding() {
-        Rounding tzRounding = Rounding.builder(TimeValue.timeValueHours(12)).build();
-        DateTimeZone tz = DateTimeZone.UTC;
-        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")), isDate(time("2009-02-03T12:00:00.000Z"), tz));
-        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T12:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T12:00:00.000Z")), isDate(time("2009-02-04T00:00:00.000Z"), tz));
-
-        tzRounding = Rounding.builder(TimeValue.timeValueHours(48)).build();
-        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")), isDate(time("2009-02-05T00:00:00.000Z"), tz));
-        assertThat(tzRounding.round(time("2009-02-05T13:01:01")), isDate(time("2009-02-05T00:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-05T00:00:00.000Z")), isDate(time("2009-02-07T00:00:00.000Z"), tz));
-    }
-
-    /**
-     * test TimeIntervalRounding, (interval < 12h) with time zone shift
-     */
-    public void testTimeIntervalRounding() {
-        DateTimeZone tz = DateTimeZone.forOffsetHours(-1);
-        Rounding tzRounding = Rounding.builder(TimeValue.timeValueHours(6)).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2009-02-03T00:01:01")), isDate(time("2009-02-02T19:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T19:00:00.000Z")), isDate(time("2009-02-03T01:00:00.000Z"), tz));
-
-        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T13:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T13:00:00.000Z")), isDate(time("2009-02-03T19:00:00.000Z"), tz));
-    }
-
-    /**
-     * test DayIntervalRounding, (interval >= 12h) with time zone shift
-     */
-    public void testDayIntervalRounding() {
-        DateTimeZone tz = DateTimeZone.forOffsetHours(-8);
-        Rounding tzRounding = Rounding.builder(TimeValue.timeValueHours(12)).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2009-02-03T00:01:01")), isDate(time("2009-02-02T20:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T20:00:00.000Z")), isDate(time("2009-02-03T08:00:00.000Z"), tz));
-
-        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T08:00:00.000Z"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T08:00:00.000Z")), isDate(time("2009-02-03T20:00:00.000Z"), tz));
-    }
-
-    public void testDayRounding() {
-        int timezoneOffset = -2;
-        Rounding tzRounding = Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(DateTimeZone.forOffsetHours(timezoneOffset)).build();
-        assertThat(tzRounding.round(0), equalTo(0L - TimeValue.timeValueHours(24 + timezoneOffset).millis()));
-        assertThat(
-            tzRounding.nextRoundingValue(0L - TimeValue.timeValueHours(24 + timezoneOffset).millis()),
-            equalTo(TimeValue.timeValueHours(-timezoneOffset).millis())
-        );
-
-        DateTimeZone tz = DateTimeZone.forID("-08:00");
-        tzRounding = Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2012-04-01T04:15:30Z")), isDate(time("2012-03-31T08:00:00Z"), tz));
-
-        tzRounding = Rounding.builder(DateTimeUnit.MONTH_OF_YEAR).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2012-04-01T04:15:30Z")), equalTo(time("2012-03-01T08:00:00Z")));
-
-        // date in Feb-3rd, but still in Feb-2nd in -02:00 timezone
-        tz = DateTimeZone.forID("-02:00");
-        tzRounding = Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-02T02:00:00"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T02:00:00")), isDate(time("2009-02-03T02:00:00"), tz));
-
-        // date in Feb-3rd, also in -02:00 timezone
-        tzRounding = Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2009-02-03T02:01:01")), isDate(time("2009-02-03T02:00:00"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T02:00:00")), isDate(time("2009-02-04T02:00:00"), tz));
-    }
-
-    public void testTimeRounding() {
-        // hour unit
-        DateTimeZone tz = DateTimeZone.forOffsetHours(-2);
-        Rounding tzRounding = Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build();
-        assertThat(tzRounding.round(0), equalTo(0L));
-        assertThat(tzRounding.nextRoundingValue(0L), equalTo(TimeValue.timeValueHours(1L).getMillis()));
-
-        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T01:00:00"), tz));
-        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T01:00:00")), isDate(time("2009-02-03T02:00:00"), tz));
-    }
-
-    public void testTimeUnitRoundingDST() {
-        Rounding tzRounding;
-        // testing savings to non savings switch
-        DateTimeZone cet = DateTimeZone.forID("CET");
-        tzRounding = Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(cet).build();
-        assertThat(tzRounding.round(time("2014-10-26T01:01:01", cet)), isDate(time("2014-10-26T01:00:00+02:00"), cet));
-        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T01:00:00", cet)), isDate(time("2014-10-26T02:00:00+02:00"), cet));
-        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T02:00:00", cet)), isDate(time("2014-10-26T02:00:00+01:00"), cet));
-
-        // testing non savings to savings switch
-        tzRounding = Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(cet).build();
-        assertThat(tzRounding.round(time("2014-03-30T01:01:01", cet)), isDate(time("2014-03-30T01:00:00+01:00"), cet));
-        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T01:00:00", cet)), isDate(time("2014-03-30T03:00:00", cet), cet));
-        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T03:00:00", cet)), isDate(time("2014-03-30T04:00:00", cet), cet));
-
-        // testing non savings to savings switch (America/Chicago)
-        DateTimeZone chg = DateTimeZone.forID("America/Chicago");
-        Rounding tzRounding_utc = Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.UTC).build();
-        assertThat(tzRounding.round(time("2014-03-09T03:01:01", chg)), isDate(time("2014-03-09T03:00:00", chg), chg));
-
-        Rounding tzRounding_chg = Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(chg).build();
-        assertThat(tzRounding_chg.round(time("2014-03-09T03:01:01", chg)), isDate(time("2014-03-09T03:00:00", chg), chg));
-
-        // testing savings to non savings switch 2013 (America/Chicago)
-        assertThat(tzRounding_utc.round(time("2013-11-03T06:01:01", chg)), isDate(time("2013-11-03T06:00:00", chg), chg));
-        assertThat(tzRounding_chg.round(time("2013-11-03T06:01:01", chg)), isDate(time("2013-11-03T06:00:00", chg), chg));
-
-        // testing savings to non savings switch 2014 (America/Chicago)
-        assertThat(tzRounding_utc.round(time("2014-11-02T06:01:01", chg)), isDate(time("2014-11-02T06:00:00", chg), chg));
-        assertThat(tzRounding_chg.round(time("2014-11-02T06:01:01", chg)), isDate(time("2014-11-02T06:00:00", chg), chg));
-    }
-
-    /**
-     * Randomized test on TimeUnitRounding. Test uses random
-     * {@link DateTimeUnit} and {@link DateTimeZone} and often (50% of the time)
-     * chooses test dates that are exactly on or close to offset changes (e.g.
-     * DST) in the chosen time zone.
-     * 
                              
-     * It rounds the test date down and up and performs various checks on the
-     * rounding unit interval that is defined by this. Assumptions tested are
-     * described in
-     * {@link #assertInterval(long, long, long, Rounding, DateTimeZone)}
-     */
-    public void testRoundingRandom() {
-        for (int i = 0; i < 1000; ++i) {
-            DateTimeUnit timeUnit = randomTimeUnit();
-            DateTimeZone tz = randomDateTimeZone();
-            Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
-            long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
-            long unitMillis = timeUnit.field(tz).getDurationField().getUnitMillis();
-            if (randomBoolean()) {
-                nastyDate(date, tz, unitMillis);
-            }
-            final long roundedDate = rounding.round(date);
-            final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
-
-            assertInterval(roundedDate, date, nextRoundingValue, rounding, tz);
-
-            // check correct unit interval width for units smaller than a day, they should be fixed size except for transitions
-            if (unitMillis <= DateTimeConstants.MILLIS_PER_DAY) {
-                // if the interval defined didn't cross timezone offset transition, it should cover unitMillis width
-                if (tz.getOffset(roundedDate - 1) == tz.getOffset(nextRoundingValue + 1)) {
-                    assertThat(
-                        "unit interval width not as expected for [" + timeUnit + "], [" + tz + "] at " + new DateTime(roundedDate),
-                        nextRoundingValue - roundedDate,
-                        equalTo(unitMillis)
-                    );
-                }
-            }
-        }
-    }
-
-    /**
-     * To be even more nasty, go to a transition in the selected time zone.
-     * In one third of the cases stay there, otherwise go half a unit back or forth
-     */
-    private static long nastyDate(long initialDate, DateTimeZone timezone, long unitMillis) {
-        long date = timezone.nextTransition(initialDate);
-        if (randomBoolean()) {
-            return date + (randomLong() % unitMillis);  // positive and negative offset possible
-        } else {
-            return date;
-        }
-    }
-
-    /**
-     * test DST end with interval rounding
-     * CET: 25 October 2015, 03:00:00 clocks were turned backward 1 hour to 25 October 2015, 02:00:00 local standard time
-     */
-    public void testTimeIntervalCET_DST_End() {
-        long interval = TimeUnit.MINUTES.toMillis(20);
-        DateTimeZone tz = DateTimeZone.forID("CET");
-        Rounding rounding = new TimeIntervalRounding(interval, tz);
-
-        assertThat(rounding.round(time("2015-10-25T01:55:00+02:00")), isDate(time("2015-10-25T01:40:00+02:00"), tz));
-        assertThat(rounding.round(time("2015-10-25T02:15:00+02:00")), isDate(time("2015-10-25T02:00:00+02:00"), tz));
-        assertThat(rounding.round(time("2015-10-25T02:35:00+02:00")), isDate(time("2015-10-25T02:20:00+02:00"), tz));
-        assertThat(rounding.round(time("2015-10-25T02:55:00+02:00")), isDate(time("2015-10-25T02:40:00+02:00"), tz));
-        // after DST shift
-        assertThat(rounding.round(time("2015-10-25T02:15:00+01:00")), isDate(time("2015-10-25T02:00:00+01:00"), tz));
-        assertThat(rounding.round(time("2015-10-25T02:35:00+01:00")), isDate(time("2015-10-25T02:20:00+01:00"), tz));
-        assertThat(rounding.round(time("2015-10-25T02:55:00+01:00")), isDate(time("2015-10-25T02:40:00+01:00"), tz));
-        assertThat(rounding.round(time("2015-10-25T03:15:00+01:00")), isDate(time("2015-10-25T03:00:00+01:00"), tz));
-    }
-
-    /**
-     * test DST start with interval rounding
-     * CET: 27 March 2016, 02:00:00 clocks were turned forward 1 hour to 27 March 2016, 03:00:00 local daylight time
-     */
-    public void testTimeIntervalCET_DST_Start() {
-        long interval = TimeUnit.MINUTES.toMillis(20);
-        DateTimeZone tz = DateTimeZone.forID("CET");
-        Rounding rounding = new TimeIntervalRounding(interval, tz);
-        // test DST start
-        assertThat(rounding.round(time("2016-03-27T01:55:00+01:00")), isDate(time("2016-03-27T01:40:00+01:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T02:00:00+01:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T03:15:00+02:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T03:35:00+02:00")), isDate(time("2016-03-27T03:20:00+02:00"), tz));
-    }
-
-    /**
-     * test DST start with offset not fitting interval, e.g. Asia/Kathmandu
-     * adding 15min on 1986-01-01T00:00:00 the interval from
-     * 1986-01-01T00:15:00+05:45 to 1986-01-01T00:20:00+05:45 to only be 5min
-     * long
-     */
-    public void testTimeInterval_Kathmandu_DST_Start() {
-        long interval = TimeUnit.MINUTES.toMillis(20);
-        DateTimeZone tz = DateTimeZone.forID("Asia/Kathmandu");
-        Rounding rounding = new TimeIntervalRounding(interval, tz);
-        assertThat(rounding.round(time("1985-12-31T23:55:00+05:30")), isDate(time("1985-12-31T23:40:00+05:30"), tz));
-        assertThat(rounding.round(time("1986-01-01T00:16:00+05:45")), isDate(time("1986-01-01T00:15:00+05:45"), tz));
-        assertThat(time("1986-01-01T00:15:00+05:45") - time("1985-12-31T23:40:00+05:30"), equalTo(TimeUnit.MINUTES.toMillis(20)));
-        assertThat(rounding.round(time("1986-01-01T00:26:00+05:45")), isDate(time("1986-01-01T00:20:00+05:45"), tz));
-        assertThat(time("1986-01-01T00:20:00+05:45") - time("1986-01-01T00:15:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(5)));
-        assertThat(rounding.round(time("1986-01-01T00:46:00+05:45")), isDate(time("1986-01-01T00:40:00+05:45"), tz));
-        assertThat(time("1986-01-01T00:40:00+05:45") - time("1986-01-01T00:20:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(20)));
-    }
-
-    /**
-     * Special test for intervals that don't fit evenly into rounding interval.
-     * In this case, when interval crosses DST transition point, rounding in local
-     * time can land in a DST gap which results in wrong UTC rounding values.
-     */
-    public void testIntervalRounding_NotDivisibleInteval() {
-        DateTimeZone tz = DateTimeZone.forID("CET");
-        long interval = TimeUnit.MINUTES.toMillis(14);
-        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
-
-        assertThat(rounding.round(time("2016-03-27T01:41:00+01:00")), isDate(time("2016-03-27T01:30:00+01:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T01:51:00+01:00")), isDate(time("2016-03-27T01:44:00+01:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T01:59:00+01:00")), isDate(time("2016-03-27T01:58:00+01:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T03:05:00+02:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T03:12:00+02:00")), isDate(time("2016-03-27T03:08:00+02:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T03:25:00+02:00")), isDate(time("2016-03-27T03:22:00+02:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T03:39:00+02:00")), isDate(time("2016-03-27T03:36:00+02:00"), tz));
-    }
-
-    /**
-     * Test for half day rounding intervals scrossing DST.
-     */
-    public void testIntervalRounding_HalfDay_DST() {
-        DateTimeZone tz = DateTimeZone.forID("CET");
-        long interval = TimeUnit.HOURS.toMillis(12);
-        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
-
-        assertThat(rounding.round(time("2016-03-26T01:00:00+01:00")), isDate(time("2016-03-26T00:00:00+01:00"), tz));
-        assertThat(rounding.round(time("2016-03-26T13:00:00+01:00")), isDate(time("2016-03-26T12:00:00+01:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T01:00:00+01:00")), isDate(time("2016-03-27T00:00:00+01:00"), tz));
-        assertThat(rounding.round(time("2016-03-27T13:00:00+02:00")), isDate(time("2016-03-27T12:00:00+02:00"), tz));
-        assertThat(rounding.round(time("2016-03-28T01:00:00+02:00")), isDate(time("2016-03-28T00:00:00+02:00"), tz));
-        assertThat(rounding.round(time("2016-03-28T13:00:00+02:00")), isDate(time("2016-03-28T12:00:00+02:00"), tz));
-    }
-
-    /**
-     * randomized test on {@link TimeIntervalRounding} with random interval and time zone offsets
-     */
-    public void testIntervalRoundingRandom() {
-        for (int i = 0; i < 1000; i++) {
-            TimeUnit unit = randomFrom(new TimeUnit[] { TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS });
-            long interval = unit.toMillis(randomIntBetween(1, 365));
-            DateTimeZone tz = randomDateTimeZone();
-            Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
-            long mainDate = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
-            if (randomBoolean()) {
-                mainDate = nastyDate(mainDate, tz, interval);
-            }
-            // check two intervals around date
-            long previousRoundedValue = Long.MIN_VALUE;
-            for (long date = mainDate - 2 * interval; date < mainDate + 2 * interval; date += interval / 2) {
-                try {
-                    final long roundedDate = rounding.round(date);
-                    final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
-                    assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate)));
-                    assertThat("Rounded value smaller or equal than unrounded", roundedDate, lessThanOrEqualTo(date));
-                    assertThat(
-                        "Values smaller than rounded value should round further down",
-                        rounding.round(roundedDate - 1),
-                        lessThan(roundedDate)
-                    );
-                    assertThat("Rounding should be >= previous rounding value", roundedDate, greaterThanOrEqualTo(previousRoundedValue));
-
-                    if (tz.isFixed()) {
-                        assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate));
-                        assertThat(
-                            "NextRounding value should be interval from rounded value",
-                            nextRoundingValue - roundedDate,
-                            equalTo(interval)
-                        );
-                        assertThat(
-                            "NextRounding value should be a rounded date",
-                            nextRoundingValue,
-                            equalTo(rounding.round(nextRoundingValue))
-                        );
-                    }
-                    previousRoundedValue = roundedDate;
-                } catch (AssertionError e) {
-                    logger.error("Rounding error at {}, timezone {}, interval: {},", new DateTime(date, tz), tz, interval);
-                    throw e;
-                }
-            }
-        }
-    }
-
-    /**
-     * Test that rounded values are always greater or equal to last rounded value if date is increasing.
-     * The example covers an interval around 2011-10-30T02:10:00+01:00, time zone CET, interval: 2700000ms
-     */
-    public void testIntervalRoundingMonotonic_CET() {
-        long interval = TimeUnit.MINUTES.toMillis(45);
-        DateTimeZone tz = DateTimeZone.forID("CET");
-        Rounding rounding = new Rounding.TimeIntervalRounding(interval, tz);
-        List> expectedDates = new ArrayList<>();
-        // first date is the date to be rounded, second the expected result
-        expectedDates.add(new Tuple<>("2011-10-30T01:40:00.000+02:00", "2011-10-30T01:30:00.000+02:00"));
-        expectedDates.add(new Tuple<>("2011-10-30T02:02:30.000+02:00", "2011-10-30T01:30:00.000+02:00"));
-        expectedDates.add(new Tuple<>("2011-10-30T02:25:00.000+02:00", "2011-10-30T02:15:00.000+02:00"));
-        expectedDates.add(new Tuple<>("2011-10-30T02:47:30.000+02:00", "2011-10-30T02:15:00.000+02:00"));
-        expectedDates.add(new Tuple<>("2011-10-30T02:10:00.000+01:00", "2011-10-30T02:15:00.000+02:00"));
-        expectedDates.add(new Tuple<>("2011-10-30T02:32:30.000+01:00", "2011-10-30T02:15:00.000+01:00"));
-        expectedDates.add(new Tuple<>("2011-10-30T02:55:00.000+01:00", "2011-10-30T02:15:00.000+01:00"));
-        expectedDates.add(new Tuple<>("2011-10-30T03:17:30.000+01:00", "2011-10-30T03:00:00.000+01:00"));
-
-        long previousDate = Long.MIN_VALUE;
-        for (Tuple dates : expectedDates) {
-            final long roundedDate = rounding.round(time(dates.v1()));
-            assertThat(roundedDate, isDate(time(dates.v2()), tz));
-            assertThat(roundedDate, greaterThanOrEqualTo(previousDate));
-            previousDate = roundedDate;
-        }
-        // here's what this means for interval widths
-        assertEquals(TimeUnit.MINUTES.toMillis(45), time("2011-10-30T02:15:00.000+02:00") - time("2011-10-30T01:30:00.000+02:00"));
-        assertEquals(TimeUnit.MINUTES.toMillis(60), time("2011-10-30T02:15:00.000+01:00") - time("2011-10-30T02:15:00.000+02:00"));
-        assertEquals(TimeUnit.MINUTES.toMillis(45), time("2011-10-30T03:00:00.000+01:00") - time("2011-10-30T02:15:00.000+01:00"));
-    }
-
-    /**
-     * special test for DST switch from #9491
-     */
-    public void testAmbiguousHoursAfterDSTSwitch() {
-        Rounding tzRounding;
-        final DateTimeZone tz = DateTimeZone.forID("Asia/Jerusalem");
-        tzRounding = Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2014-10-26T00:30:00+03:00")), isDate(time("2014-10-26T00:00:00+03:00"), tz));
-        assertThat(tzRounding.round(time("2014-10-26T01:30:00+03:00")), isDate(time("2014-10-26T01:00:00+03:00"), tz));
-        // the utc date for "2014-10-25T03:00:00+03:00" and "2014-10-25T03:00:00+02:00" is the same, local time turns back 1h here
-        assertThat(time("2014-10-26T03:00:00+03:00"), isDate(time("2014-10-26T02:00:00+02:00"), tz));
-        assertThat(tzRounding.round(time("2014-10-26T01:30:00+02:00")), isDate(time("2014-10-26T01:00:00+02:00"), tz));
-        assertThat(tzRounding.round(time("2014-10-26T02:30:00+02:00")), isDate(time("2014-10-26T02:00:00+02:00"), tz));
-
-        // Day interval
-        tzRounding = Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-11-11T00:00:00", tz), tz));
-        // DST on
-        assertThat(tzRounding.round(time("2014-08-11T17:00:00", tz)), isDate(time("2014-08-11T00:00:00", tz), tz));
-        // Day of switching DST on -> off
-        assertThat(tzRounding.round(time("2014-10-26T17:00:00", tz)), isDate(time("2014-10-26T00:00:00", tz), tz));
-        // Day of switching DST off -> on
-        assertThat(tzRounding.round(time("2015-03-27T17:00:00", tz)), isDate(time("2015-03-27T00:00:00", tz), tz));
-
-        // Month interval
-        tzRounding = Rounding.builder(DateTimeUnit.MONTH_OF_YEAR).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-11-01T00:00:00", tz), tz));
-        // DST on
-        assertThat(tzRounding.round(time("2014-10-10T17:00:00", tz)), isDate(time("2014-10-01T00:00:00", tz), tz));
-
-        // Year interval
-        tzRounding = Rounding.builder(DateTimeUnit.YEAR_OF_CENTURY).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-01-01T00:00:00", tz), tz));
-
-        // Two timestamps in same year and different timezone offset ("Double buckets" issue - #9491)
-        tzRounding = Rounding.builder(DateTimeUnit.YEAR_OF_CENTURY).timeZone(tz).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(tzRounding.round(time("2014-08-11T17:00:00", tz)), tz));
-    }
-
-    /**
-     * test for #10025, strict local to UTC conversion can cause joda exceptions
-     * on DST start
-     */
-    public void testLenientConversionDST() {
-        DateTimeZone tz = DateTimeZone.forID("America/Sao_Paulo");
-        long start = time("2014-10-18T20:50:00.000", tz);
-        long end = time("2014-10-19T01:00:00.000", tz);
-        Rounding tzRounding = new Rounding.TimeUnitRounding(DateTimeUnit.MINUTES_OF_HOUR, tz);
-        Rounding dayTzRounding = new Rounding.TimeIntervalRounding(60000, tz);
-        for (long time = start; time < end; time = time + 60000) {
-            assertThat(tzRounding.nextRoundingValue(time), greaterThan(time));
-            assertThat(dayTzRounding.nextRoundingValue(time), greaterThan(time));
-        }
-    }
-
-    public void testEdgeCasesTransition() {
-        {
-            // standard +/-1 hour DST transition, CET
-            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone tz = DateTimeZone.forID("CET");
-            Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
-
-            // 29 Mar 2015 - Daylight Saving Time Started
-            // at 02:00:00 clocks were turned forward 1 hour to 03:00:00
-            assertInterval(time("2015-03-29T00:00:00.000+01:00"), time("2015-03-29T01:00:00.000+01:00"), rounding, 60, tz);
-            assertInterval(time("2015-03-29T01:00:00.000+01:00"), time("2015-03-29T03:00:00.000+02:00"), rounding, 60, tz);
-            assertInterval(time("2015-03-29T03:00:00.000+02:00"), time("2015-03-29T04:00:00.000+02:00"), rounding, 60, tz);
-
-            // 25 Oct 2015 - Daylight Saving Time Ended
-            // at 03:00:00 clocks were turned backward 1 hour to 02:00:00
-            assertInterval(time("2015-10-25T01:00:00.000+02:00"), time("2015-10-25T02:00:00.000+02:00"), rounding, 60, tz);
-            assertInterval(time("2015-10-25T02:00:00.000+02:00"), time("2015-10-25T02:00:00.000+01:00"), rounding, 60, tz);
-            assertInterval(time("2015-10-25T02:00:00.000+01:00"), time("2015-10-25T03:00:00.000+01:00"), rounding, 60, tz);
-        }
-
-        {
-            // time zone "Asia/Kathmandu"
-            // 1 Jan 1986 - Time Zone Change (IST → NPT), at 00:00:00 clocks were turned forward 00:15 minutes
-            //
-            // hour rounding is stable before 1985-12-31T23:00:00.000 and after 1986-01-01T01:00:00.000+05:45
-            // the interval between is 105 minutes long because the hour after transition starts at 00:15
-            // which is not a round value for hourly rounding
-            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone tz = DateTimeZone.forID("Asia/Kathmandu");
-            Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
-
-            assertInterval(time("1985-12-31T22:00:00.000+05:30"), time("1985-12-31T23:00:00.000+05:30"), rounding, 60, tz);
-            assertInterval(time("1985-12-31T23:00:00.000+05:30"), time("1986-01-01T01:00:00.000+05:45"), rounding, 105, tz);
-            assertInterval(time("1986-01-01T01:00:00.000+05:45"), time("1986-01-01T02:00:00.000+05:45"), rounding, 60, tz);
-        }
-
-        {
-            // time zone "Australia/Lord_Howe"
-            // 3 Mar 1991 - Daylight Saving Time Ended
-            // at 02:00:00 clocks were turned backward 0:30 hours to Sunday, 3 March 1991, 01:30:00
-            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone tz = DateTimeZone.forID("Australia/Lord_Howe");
-            Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
-
-            assertInterval(time("1991-03-03T00:00:00.000+11:00"), time("1991-03-03T01:00:00.000+11:00"), rounding, 60, tz);
-            assertInterval(time("1991-03-03T01:00:00.000+11:00"), time("1991-03-03T02:00:00.000+10:30"), rounding, 90, tz);
-            assertInterval(time("1991-03-03T02:00:00.000+10:30"), time("1991-03-03T03:00:00.000+10:30"), rounding, 60, tz);
-
-            // 27 Oct 1991 - Daylight Saving Time Started
-            // at 02:00:00 clocks were turned forward 0:30 hours to 02:30:00
-            assertInterval(time("1991-10-27T00:00:00.000+10:30"), time("1991-10-27T01:00:00.000+10:30"), rounding, 60, tz);
-            // the interval containing the switch time is 90 minutes long
-            assertInterval(time("1991-10-27T01:00:00.000+10:30"), time("1991-10-27T03:00:00.000+11:00"), rounding, 90, tz);
-            assertInterval(time("1991-10-27T03:00:00.000+11:00"), time("1991-10-27T04:00:00.000+11:00"), rounding, 60, tz);
-        }
-
-        {
-            // time zone "Pacific/Chatham"
-            // 5 Apr 2015 - Daylight Saving Time Ended
-            // at 03:45:00 clocks were turned backward 1 hour to 02:45:00
-            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone tz = DateTimeZone.forID("Pacific/Chatham");
-            Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
-
-            assertInterval(time("2015-04-05T02:00:00.000+13:45"), time("2015-04-05T03:00:00.000+13:45"), rounding, 60, tz);
-            assertInterval(time("2015-04-05T03:00:00.000+13:45"), time("2015-04-05T03:00:00.000+12:45"), rounding, 60, tz);
-            assertInterval(time("2015-04-05T03:00:00.000+12:45"), time("2015-04-05T04:00:00.000+12:45"), rounding, 60, tz);
-
-            // 27 Sep 2015 - Daylight Saving Time Started
-            // at 02:45:00 clocks were turned forward 1 hour to 03:45:00
-
-            assertInterval(time("2015-09-27T01:00:00.000+12:45"), time("2015-09-27T02:00:00.000+12:45"), rounding, 60, tz);
-            assertInterval(time("2015-09-27T02:00:00.000+12:45"), time("2015-09-27T04:00:00.000+13:45"), rounding, 60, tz);
-            assertInterval(time("2015-09-27T04:00:00.000+13:45"), time("2015-09-27T05:00:00.000+13:45"), rounding, 60, tz);
-        }
-    }
-
-    public void testDST_Europe_Rome() {
-        // time zone "Europe/Rome", rounding to days. Rome had two midnights on the day the clocks went back in 1978, and
-        // timeZone.convertLocalToUTC() gives the later of the two because Rome is east of UTC, whereas we want the earlier.
-
-        DateTimeUnit timeUnit = DateTimeUnit.DAY_OF_MONTH;
-        DateTimeZone tz = DateTimeZone.forID("Europe/Rome");
-        Rounding rounding = new TimeUnitRounding(timeUnit, tz);
-
-        {
-            long timeBeforeFirstMidnight = time("1978-09-30T23:59:00+02:00");
-            long floor = rounding.round(timeBeforeFirstMidnight);
-            assertThat(floor, isDate(time("1978-09-30T00:00:00+02:00"), tz));
-        }
-
-        {
-            long timeBetweenMidnights = time("1978-10-01T00:30:00+02:00");
-            long floor = rounding.round(timeBetweenMidnights);
-            assertThat(floor, isDate(time("1978-10-01T00:00:00+02:00"), tz));
-        }
-
-        {
-            long timeAfterSecondMidnight = time("1978-10-01T00:30:00+01:00");
-            long floor = rounding.round(timeAfterSecondMidnight);
-            assertThat(floor, isDate(time("1978-10-01T00:00:00+02:00"), tz));
-
-            long prevFloor = rounding.round(floor - 1);
-            assertThat(prevFloor, lessThan(floor));
-            assertThat(prevFloor, isDate(time("1978-09-30T00:00:00+02:00"), tz));
-        }
-    }
-
-    /**
-     * Test for a time zone whose days overlap because the clocks are set back across midnight at the end of DST.
-     */
-    public void testDST_America_St_Johns() {
-        // time zone "America/St_Johns", rounding to days.
-        DateTimeUnit timeUnit = DateTimeUnit.DAY_OF_MONTH;
-        DateTimeZone tz = DateTimeZone.forID("America/St_Johns");
-        Rounding rounding = new TimeUnitRounding(timeUnit, tz);
-
-        // 29 October 2006 - Daylight Saving Time ended, changing the UTC offset from -02:30 to -03:30.
-        // This happened at 02:31 UTC, 00:01 local time, so the clocks were set back 1 hour to 23:01 on the 28th.
-        // This means that 2006-10-29 has _two_ midnights, one in the -02:30 offset and one in the -03:30 offset.
-        // Only the first of these is considered "rounded". Moreover, the extra time between 23:01 and 23:59
-        // should be considered as part of the 28th even though it comes after midnight on the 29th.
-
-        {
-            // Times before the first midnight should be rounded up to the first midnight.
-            long timeBeforeFirstMidnight = time("2006-10-28T23:30:00.000-02:30");
-            long floor = rounding.round(timeBeforeFirstMidnight);
-            assertThat(floor, isDate(time("2006-10-28T00:00:00.000-02:30"), tz));
-            long ceiling = rounding.nextRoundingValue(timeBeforeFirstMidnight);
-            assertThat(ceiling, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
-            assertInterval(floor, timeBeforeFirstMidnight, ceiling, rounding, tz);
-        }
-
-        {
-            // Times between the two midnights which are on the later day should be rounded down to the later day's midnight.
-            long timeBetweenMidnights = time("2006-10-29T00:00:30.000-02:30");
-            // (this is halfway through the last minute before the clocks changed, in which local time was ambiguous)
-
-            long floor = rounding.round(timeBetweenMidnights);
-            assertThat(floor, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
-
-            long ceiling = rounding.nextRoundingValue(timeBetweenMidnights);
-            assertThat(ceiling, isDate(time("2006-10-30T00:00:00.000-03:30"), tz));
-
-            assertInterval(floor, timeBetweenMidnights, ceiling, rounding, tz);
-        }
-
-        {
-            // Times between the two midnights which are on the earlier day should be rounded down to the earlier day's midnight.
-            long timeBetweenMidnights = time("2006-10-28T23:30:00.000-03:30");
-            // (this is halfway through the hour after the clocks changed, in which local time was ambiguous)
-
-            long floor = rounding.round(timeBetweenMidnights);
-            assertThat(floor, isDate(time("2006-10-28T00:00:00.000-02:30"), tz));
-
-            long ceiling = rounding.nextRoundingValue(timeBetweenMidnights);
-            assertThat(ceiling, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
-
-            assertInterval(floor, timeBetweenMidnights, ceiling, rounding, tz);
-        }
-
-        {
-            // Times after the second midnight should be rounded down to the first midnight.
-            long timeAfterSecondMidnight = time("2006-10-29T06:00:00.000-03:30");
-            long floor = rounding.round(timeAfterSecondMidnight);
-            assertThat(floor, isDate(time("2006-10-29T00:00:00.000-02:30"), tz));
-            long ceiling = rounding.nextRoundingValue(timeAfterSecondMidnight);
-            assertThat(ceiling, isDate(time("2006-10-30T00:00:00.000-03:30"), tz));
-            assertInterval(floor, timeAfterSecondMidnight, ceiling, rounding, tz);
-        }
-    }
-
-    /**
-     * tests for dst transition with overlaps and day roundings.
-     */
-    public void testDST_END_Edgecases() {
-        // First case, dst happens at 1am local time, switching back one hour.
-        // We want the overlapping hour to count for the next day, making it a 25h interval
-
-        DateTimeUnit timeUnit = DateTimeUnit.DAY_OF_MONTH;
-        DateTimeZone tz = DateTimeZone.forID("Atlantic/Azores");
-        Rounding rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
-
-        // Sunday, 29 October 2000, 01:00:00 clocks were turned backward 1 hour
-        // to Sunday, 29 October 2000, 00:00:00 local standard time instead
-        // which means there were two midnights that day.
-
-        long midnightBeforeTransition = time("2000-10-29T00:00:00", tz);
-        long midnightOfTransition = time("2000-10-29T00:00:00-01:00");
-        assertEquals(60L * 60L * 1000L, midnightOfTransition - midnightBeforeTransition);
-        long nextMidnight = time("2000-10-30T00:00:00", tz);
-
-        assertInterval(midnightBeforeTransition, nextMidnight, rounding, 25 * 60, tz);
-
-        assertThat(rounding.round(time("2000-10-29T06:00:00-01:00")), isDate(time("2000-10-29T00:00:00Z"), tz));
-
-        // Second case, dst happens at 0am local time, switching back one hour to 23pm local time.
-        // We want the overlapping hour to count for the previous day here
-
-        tz = DateTimeZone.forID("America/Lima");
-        rounding = new Rounding.TimeUnitRounding(timeUnit, tz);
-
-        // Sunday, 1 April 1990, 00:00:00 clocks were turned backward 1 hour to
-        // Saturday, 31 March 1990, 23:00:00 local standard time instead
-
-        midnightBeforeTransition = time("1990-03-31T00:00:00.000-04:00");
-        nextMidnight = time("1990-04-01T00:00:00.000-05:00");
-        assertInterval(midnightBeforeTransition, nextMidnight, rounding, 25 * 60, tz);
-
-        // make sure the next interval is 24h long again
-        long midnightAfterTransition = time("1990-04-01T00:00:00.000-05:00");
-        nextMidnight = time("1990-04-02T00:00:00.000-05:00");
-        assertInterval(midnightAfterTransition, nextMidnight, rounding, 24 * 60, tz);
-    }
-
-    /**
-     * Test that time zones are correctly parsed. There is a bug with
-     * Joda 2.9.4 (see https://github.com/JodaOrg/joda-time/issues/373)
-     */
-    public void testsTimeZoneParsing() {
-        final DateTime expected = new DateTime(2016, 11, 10, 5, 37, 59, randomDateTimeZone());
-
-        // Formatter used to print and parse the sample date.
-        // Printing the date works but parsing it back fails
-        // with Joda 2.9.4
-        DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYY-MM-dd'T'HH:mm:ss " + randomFrom("ZZZ", "[ZZZ]", "'['ZZZ']'"));
-
-        String dateTimeAsString = formatter.print(expected);
-        assertThat(dateTimeAsString, startsWith("2016-11-10T05:37:59 "));
-
-        DateTime parsedDateTime = formatter.parseDateTime(dateTimeAsString);
-        assertThat(parsedDateTime.getZone(), equalTo(expected.getZone()));
-    }
-
-    private static void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes, DateTimeZone tz) {
-        assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz);
-        assertEquals(DateTimeConstants.MILLIS_PER_MINUTE * minutes, nextRoundingValue - rounded);
-    }
-
-    /**
-     * perform a number on assertions and checks on {@link TimeUnitRounding} intervals
-     * @param rounded the expected low end of the rounding interval
-     * @param unrounded a date in the interval to be checked for rounding
-     * @param nextRoundingValue the expected upper end of the rounding interval
-     * @param rounding the rounding instance
-     */
-    private static void assertInterval(long rounded, long unrounded, long nextRoundingValue, Rounding rounding, DateTimeZone tz) {
-        assertThat("rounding should be idempotent ", rounding.round(rounded), isDate(rounded, tz));
-        assertThat("rounded value smaller or equal than unrounded" + rounding, rounded, lessThanOrEqualTo(unrounded));
-        assertThat("values less than rounded should round further down" + rounding, rounding.round(rounded - 1), lessThan(rounded));
-        assertThat("nextRounding value should be a rounded date", rounding.round(nextRoundingValue), isDate(nextRoundingValue, tz));
-        assertThat(
-            "values above nextRounding should round down there",
-            rounding.round(nextRoundingValue + 1),
-            isDate(nextRoundingValue, tz)
-        );
-
-        if (isTimeWithWellDefinedRounding(tz, unrounded)) {
-            assertThat("nextRounding value should be greater than date" + rounding, nextRoundingValue, greaterThan(unrounded));
-
-            long dateBetween = dateBetween(rounded, nextRoundingValue);
-            assertThat(
-                "dateBetween [" + new DateTime(dateBetween, tz) + "] should round down to roundedDate",
-                rounding.round(dateBetween),
-                isDate(rounded, tz)
-            );
-            assertThat(
-                "dateBetween [" + new DateTime(dateBetween, tz) + "] should round up to nextRoundingValue",
-                rounding.nextRoundingValue(dateBetween),
-                isDate(nextRoundingValue, tz)
-            );
-        }
-    }
-
-    private static boolean isTimeWithWellDefinedRounding(DateTimeZone tz, long t) {
-        if (tz.getID().equals("America/St_Johns")
-            || tz.getID().equals("America/Goose_Bay")
-            || tz.getID().equals("America/Moncton")
-            || tz.getID().equals("Canada/Newfoundland")) {
-
-            // Clocks went back at 00:01 between 1987 and 2010, causing overlapping days.
-            // These timezones are otherwise uninteresting, so just skip this period.
-
-            return t <= time("1987-10-01T00:00:00Z") || t >= time("2010-12-01T00:00:00Z");
-        }
-
-        if (tz.getID().equals("Antarctica/Casey")) {
-
-            // Clocks went back 3 hours at 02:00 on 2010-03-05, causing overlapping days.
-
-            return t <= time("2010-03-03T00:00:00Z") || t >= time("2010-03-07T00:00:00Z");
-        }
-
-        return true;
-    }
-
-    private static long dateBetween(long lower, long upper) {
-        long dateBetween = randomLongBetween(lower, upper - 1);
-        assert lower <= dateBetween && dateBetween < upper;
-        return dateBetween;
-    }
-
-    private static DateTimeUnit randomTimeUnit() {
-        byte id = (byte) randomIntBetween(1, 8);
-        return DateTimeUnit.resolve(id);
-    }
-
-    private static long time(String time) {
-        return time(time, DateTimeZone.UTC);
-    }
-
-    private static long time(String time, DateTimeZone zone) {
-        return ISODateTimeFormat.dateOptionalTimeParser().withZone(zone).parseMillis(time);
-    }
-
-    private static Matcher isDate(final long expected, DateTimeZone tz) {
-        return new TypeSafeMatcher() {
-            @Override
-            public boolean matchesSafely(final Long item) {
-                return expected == item.longValue();
-            }
-
-            @Override
-            public void describeTo(Description description) {
-                description.appendText(new DateTime(expected, tz) + " [" + expected + "] ");
-            }
-
-            @Override
-            protected void describeMismatchSafely(final Long actual, final Description mismatchDescription) {
-                mismatchDescription.appendText(" was ").appendValue(new DateTime(actual, tz) + " [" + actual + "]");
-            }
-        };
-    }
-}

From fcbec24408147f9b55b71458996c1cdb6a42dfbd Mon Sep 17 00:00:00 2001
From: Siddhant Deshmukh 
Date: Fri, 27 Oct 2023 12:50:35 -0700
Subject: [PATCH 246/388] Change log level to trace (#10971)

Signed-off-by: Siddhant Deshmukh 
---
 .../org/opensearch/action/search/SearchQueryCategorizer.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
index 9cbe2d2ffcb7d..8fe1be610f9af 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
@@ -75,7 +75,7 @@ private void logQueryShape(QueryBuilder topLevelQueryBuilder) {
         }
         QueryShapeVisitor shapeVisitor = new QueryShapeVisitor();
         topLevelQueryBuilder.visit(shapeVisitor);
-        log.debug("Query shape : {}", shapeVisitor.prettyPrintTree("  "));
+        log.trace("Query shape : {}", shapeVisitor.prettyPrintTree("  "));
     }
 
 }

From bc74731a537c461cd1d7666bc3be82fbe18e143d Mon Sep 17 00:00:00 2001
From: Neetika Singhal 
Date: Fri, 27 Oct 2023 16:41:26 -0700
Subject: [PATCH 247/388] update the indexRandom function to create more
 segments and update (#10247)

IndicesRequestCacheIT, HighlighterSearcherIT to run with minimum of 2
slices

Signed-off-by: Neetika Singhal 
---
 CHANGELOG.md                                  |   1 +
 .../indices/IndicesRequestCacheIT.java        |   4 +-
 .../highlight/HighlighterSearchIT.java        | 134 ++++++++++++++----
 .../test/OpenSearchIntegTestCase.java         |  51 +++++++
 .../ParameterizedOpenSearchIntegTestCase.java |   8 ++
 5 files changed, 168 insertions(+), 30 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 234b08398f9ef..34fd573b295b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -95,6 +95,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add repository stats for remote store([#10567](https://github.com/opensearch-project/OpenSearch/pull/10567))
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
+- Update the indexRandom function to create more segments for concurrent search tests ([10247](https://github.com/opensearch-project/OpenSearch/pull/10247))
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
 - Added cluster setting cluster.restrict.index.replication_type to restrict setting of index setting replication type ([#10866](https://github.com/opensearch-project/OpenSearch/pull/10866))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 98a22717019cf..848f6eddbb0df 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -539,7 +539,7 @@ public void testCanCache() throws Exception {
         assertCacheState(client, "index", 0, 4);
     }
 
-    public void testCacheWithFilteredAlias() {
+    public void testCacheWithFilteredAlias() throws InterruptedException {
         Client client = client();
         Settings settings = Settings.builder()
             .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
@@ -562,6 +562,8 @@ public void testCacheWithFilteredAlias() {
         OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
         refresh();
 
+        indexRandomForConcurrentSearch("index");
+
         assertCacheState(client, "index", 0, 0);
 
         SearchResponse r1 = client.prepareSearch("index")
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index 42d91ac945662..f7bc5eb75ad0f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -155,7 +155,7 @@ protected Collection> nodePlugins() {
         return Arrays.asList(InternalSettingsPlugin.class, MockKeywordPlugin.class, MockAnalysisPlugin.class);
     }
 
-    public void testHighlightingWithKeywordIgnoreBoundaryScanner() throws IOException {
+    public void testHighlightingWithKeywordIgnoreBoundaryScanner() throws IOException, InterruptedException {
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
         mappings.startObject("properties")
@@ -177,6 +177,7 @@ public void testHighlightingWithKeywordIgnoreBoundaryScanner() throws IOExceptio
             .setSource(jsonBuilder().startObject().array("tags", "foo baz", "foo baz", "foo baz", "foo bar").field("sort", 2).endObject())
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         for (BoundaryScannerType scanner : BoundaryScannerType.values()) {
             SearchResponse search = client().prepareSearch()
@@ -190,12 +191,13 @@ public void testHighlightingWithKeywordIgnoreBoundaryScanner() throws IOExceptio
         }
     }
 
-    public void testHighlightingWithStoredKeyword() throws IOException {
+    public void testHighlightingWithStoredKeyword() throws IOException, InterruptedException {
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
         mappings.startObject("properties").startObject("text").field("type", "keyword").field("store", true).endObject().endObject();
         mappings.endObject();
         assertAcked(prepareCreate("test").setMapping(mappings));
+        indexRandomForConcurrentSearch("test");
         client().prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("text", "foo").endObject()).get();
         refresh();
         SearchResponse search = client().prepareSearch()
@@ -205,7 +207,7 @@ public void testHighlightingWithStoredKeyword() throws IOException {
         assertHighlight(search, 0, "text", 0, equalTo("foo"));
     }
 
-    public void testHighlightingWithWildcardName() throws IOException {
+    public void testHighlightingWithWildcardName() throws IOException, InterruptedException {
         // test the kibana case with * as fieldname that will try highlight all fields including meta fields
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
@@ -221,6 +223,7 @@ public void testHighlightingWithWildcardName() throws IOException {
         assertAcked(prepareCreate("test").setMapping(mappings));
         client().prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("text", "text").endObject()).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         for (String type : ALL_TYPES) {
             SearchResponse search = client().prepareSearch()
                 .setQuery(constantScoreQuery(matchQuery("text", "text")))
@@ -230,7 +233,7 @@ public void testHighlightingWithWildcardName() throws IOException {
         }
     }
 
-    public void testFieldAlias() throws IOException {
+    public void testFieldAlias() throws IOException, InterruptedException {
         XContentBuilder mappings = jsonBuilder().startObject()
             .startObject("properties")
             .startObject("text")
@@ -248,7 +251,7 @@ public void testFieldAlias() throws IOException {
 
         client().prepareIndex("test").setId("1").setSource("text", "foo").get();
         refresh();
-
+        indexRandomForConcurrentSearch("test");
         for (String type : ALL_TYPES) {
             HighlightBuilder builder = new HighlightBuilder().field(new Field("alias").highlighterType(type))
                 .requireFieldMatch(randomBoolean());
@@ -257,7 +260,7 @@ public void testFieldAlias() throws IOException {
         }
     }
 
-    public void testFieldAliasWithSourceLookup() throws IOException {
+    public void testFieldAliasWithSourceLookup() throws IOException, InterruptedException {
         XContentBuilder mappings = jsonBuilder().startObject()
             .startObject("properties")
             .startObject("text")
@@ -276,7 +279,7 @@ public void testFieldAliasWithSourceLookup() throws IOException {
 
         client().prepareIndex("test").setId("1").setSource("text", "foo bar").get();
         refresh();
-
+        indexRandomForConcurrentSearch("test");
         for (String type : ALL_TYPES) {
             HighlightBuilder builder = new HighlightBuilder().field(new Field("alias").highlighterType(type))
                 .requireFieldMatch(randomBoolean());
@@ -285,7 +288,7 @@ public void testFieldAliasWithSourceLookup() throws IOException {
         }
     }
 
-    public void testFieldAliasWithWildcardField() throws IOException {
+    public void testFieldAliasWithWildcardField() throws IOException, InterruptedException {
         XContentBuilder mappings = jsonBuilder().startObject()
             .startObject("properties")
             .startObject("keyword")
@@ -301,13 +304,14 @@ public void testFieldAliasWithWildcardField() throws IOException {
 
         client().prepareIndex("test").setId("1").setSource("keyword", "foo").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         HighlightBuilder builder = new HighlightBuilder().field(new Field("al*")).requireFieldMatch(false);
         SearchResponse search = client().prepareSearch().setQuery(matchQuery("alias", "foo")).highlighter(builder).get();
         assertHighlight(search, 0, "alias", 0, equalTo("foo"));
     }
 
-    public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOException {
+    public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOException, InterruptedException {
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
         mappings.startObject("_source")
@@ -334,6 +338,7 @@ public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOExc
             .setSource(jsonBuilder().startObject().field("unstored_text", "text").field("text", "text").endObject())
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         for (String type : ALL_TYPES) {
             SearchResponse search = client().prepareSearch()
                 .setQuery(constantScoreQuery(matchQuery("text", "text")))
@@ -350,7 +355,7 @@ public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOExc
     }
 
     // see #3486
-    public void testHighTermFrequencyDoc() throws IOException {
+    public void testHighTermFrequencyDoc() throws IOException, InterruptedException {
         assertAcked(prepareCreate("test").setMapping("name", "type=text,term_vector=with_positions_offsets,store=" + randomBoolean()));
         StringBuilder builder = new StringBuilder();
         for (int i = 0; i < 6000; i++) {
@@ -358,6 +363,7 @@ public void testHighTermFrequencyDoc() throws IOException {
         }
         client().prepareIndex("test").setId("1").setSource("name", builder.toString()).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         SearchResponse search = client().prepareSearch()
             .setQuery(constantScoreQuery(matchQuery("name", "abc")))
             .highlighter(new HighlightBuilder().field("name"))
@@ -385,6 +391,7 @@ public void testEnsureNoNegativeOffsets() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         SearchResponse search = client().prepareSearch()
             .setQuery(matchQuery("long_term", "thisisaverylongwordandmakessurethisfails foo highlighed"))
             .highlighter(new HighlightBuilder().field("long_term", 18, 1).highlighterType("fvh"))
@@ -671,7 +678,7 @@ public void testHighlightIssue1994() throws Exception {
         assertHighlight(search, 0, "titleTV", 1, 2, equalTo("highlight other text"));
     }
 
-    public void testGlobalHighlightingSettingsOverriddenAtFieldLevel() {
+    public void testGlobalHighlightingSettingsOverriddenAtFieldLevel() throws InterruptedException {
         createIndex("test");
         ensureGreen();
 
@@ -684,6 +691,7 @@ public void testGlobalHighlightingSettingsOverriddenAtFieldLevel() {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1 and field2 produces different tags");
         SearchSourceBuilder source = searchSource().query(termQuery("field1", "test"))
@@ -734,6 +742,7 @@ public void testHighlightingOnWildcardFields() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field*");
         SearchSourceBuilder source = searchSource()
@@ -783,6 +792,7 @@ public void testForceSourceWithSourceDisabled() throws Exception {
             .setSource("field1", "The quick brown fox jumps over the lazy dog", "field2", "second field content")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // works using stored field
         SearchResponse searchResponse = client().prepareSearch("test")
@@ -823,6 +833,7 @@ public void testPlainHighlighter() throws Exception {
 
         client().prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(termQuery("field1", "test"))
@@ -1025,6 +1036,7 @@ public void testFVHManyMatches() throws Exception {
         String value = new String(new char[1024 * 256 / pattern.length()]).replace("\0", pattern);
         client().prepareIndex("test").setSource("field1", value).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1 with default phrase limit");
         SearchSourceBuilder source = searchSource().query(termQuery("field1", "t"))
@@ -1116,6 +1128,7 @@ private void checkMatchedFieldsCase(boolean requireFieldMatch) throws Exception
         );
         index("test", "type1", "3", "foo", "weird", "bar", "result");
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         Field fooField = new Field("foo").numOfFragments(1)
             .order("score")
@@ -1408,6 +1421,7 @@ public void testMultiMapperVectorWithStore() throws Exception {
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("title", "this is a test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // simple search on body with standard analyzer with a simple field query
         SearchResponse search = client().prepareSearch()
@@ -1453,6 +1467,7 @@ public void testMultiMapperVectorFromSource() throws Exception {
 
         client().prepareIndex("test").setId("1").setSource("title", "this is a test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // simple search on body with standard analyzer with a simple field query
         SearchResponse search = client().prepareSearch()
@@ -1498,6 +1513,7 @@ public void testMultiMapperNoVectorWithStore() throws Exception {
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("title", "this is a test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // simple search on body with standard analyzer with a simple field query
         SearchResponse search = client().prepareSearch()
@@ -1542,6 +1558,7 @@ public void testMultiMapperNoVectorFromSource() throws Exception {
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("title", "this is a test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // simple search on body with standard analyzer with a simple field query
         SearchResponse search = client().prepareSearch()
@@ -1571,6 +1588,7 @@ public void testFastVectorHighlighterShouldFailIfNoTermVectors() throws Exceptio
                 .setSource("title", "This is a test for the enabling fast vector highlighter");
         }
         indexRandom(true, indexRequestBuilders);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse search = client().prepareSearch()
             .setQuery(matchPhraseQuery("title", "this is a test"))
@@ -1608,6 +1626,7 @@ public void testDisableFastVectorHighlighter() throws Exception {
                 .setSource("title", "This is a test for the workaround for the fast vector highlighting SOLR-3724");
         }
         indexRandom(true, indexRequestBuilders);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse search = client().prepareSearch()
             .setQuery(matchPhraseQuery("title", "test for the workaround"))
@@ -1669,6 +1688,7 @@ public void testFSHHighlightAllMvFragments() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchQuery("tags", "tag"))
@@ -1686,11 +1706,12 @@ public void testFSHHighlightAllMvFragments() throws Exception {
         );
     }
 
-    public void testBoostingQuery() {
+    public void testBoostingQuery() throws InterruptedException {
         createIndex("test");
         ensureGreen();
         client().prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(
@@ -1702,11 +1723,12 @@ public void testBoostingQuery() {
         assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
     }
 
-    public void testBoostingQueryTermVector() throws IOException {
+    public void testBoostingQueryTermVector() throws IOException, InterruptedException {
         assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping()));
         ensureGreen();
         client().prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(
@@ -1718,12 +1740,13 @@ public void testBoostingQueryTermVector() throws IOException {
         assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
     }
 
-    public void testCommonTermsQuery() {
+    public void testCommonTermsQuery() throws InterruptedException {
         createIndex("test");
         ensureGreen();
 
         client().prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
@@ -1733,12 +1756,13 @@ public void testCommonTermsQuery() {
         assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog"));
     }
 
-    public void testCommonTermsTermVector() throws IOException {
+    public void testCommonTermsTermVector() throws IOException, InterruptedException {
         assertAcked(prepareCreate("test").setMapping(type1TermVectorMapping()));
         ensureGreen();
 
         client().prepareIndex("test").setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
             .highlighter(highlight().field("field2").order("score").preTags("").postTags(""));
@@ -1764,6 +1788,7 @@ public void testPlainHighlightDifferentFragmenter() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchPhraseQuery("tags", "long tag"))
@@ -1816,12 +1841,13 @@ public void testPlainHighlightDifferentFragmenter() throws Exception {
         );
     }
 
-    public void testPlainHighlighterMultipleFields() {
+    public void testPlainHighlighterMultipleFields() throws InterruptedException {
         createIndex("test");
         ensureGreen();
 
         index("test", "type1", "1", "field1", "The quick brown fox", "field2", "The slow brown fox");
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchQuery("field1", "fox"))
@@ -1834,7 +1860,7 @@ public void testPlainHighlighterMultipleFields() {
         assertHighlight(response, 0, "field2", 0, 1, equalTo("The slow brown <2>fox"));
     }
 
-    public void testFastVectorHighlighterMultipleFields() {
+    public void testFastVectorHighlighterMultipleFields() throws InterruptedException {
         assertAcked(
             prepareCreate("test").setMapping(
                 "field1",
@@ -1847,6 +1873,7 @@ public void testFastVectorHighlighterMultipleFields() {
 
         index("test", "type1", "1", "field1", "The quick brown fox", "field2", "The slow brown fox");
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchQuery("field1", "fox"))
@@ -1864,6 +1891,7 @@ public void testMissingStoredField() throws Exception {
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "highlight").endObject()).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // This query used to fail when the field to highlight was absent
         SearchResponse response = client().prepareSearch("test")
@@ -1904,6 +1932,7 @@ public void testNumericHighlighting() throws Exception {
             .setSource("text", "opensearch test", "byte", 25, "short", 42, "int", 100, "long", -1, "float", 3.2f, "double", 42.42)
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchQuery("text", "test"))
@@ -1926,6 +1955,7 @@ public void testResetTwice() throws Exception {
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("text", "opensearch test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchQuery("text", "test"))
@@ -1935,7 +1965,7 @@ public void testResetTwice() throws Exception {
         assertHitCount(response, 1L);
     }
 
-    public void testHighlightUsesHighlightQuery() throws IOException {
+    public void testHighlightUsesHighlightQuery() throws IOException, InterruptedException {
         assertAcked(
             prepareCreate("test").setMapping(
                 "text",
@@ -1946,6 +1976,7 @@ public void testHighlightUsesHighlightQuery() throws IOException {
 
         index("test", "type1", "1", "text", "Testing the highlight query feature");
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         for (String type : ALL_TYPES) {
             HighlightBuilder.Field field = new HighlightBuilder.Field("text");
@@ -1981,7 +2012,11 @@ private static String randomStoreField() {
         return "";
     }
 
-    public void testHighlightNoMatchSize() throws IOException {
+    public void testHighlightNoMatchSize() throws IOException, InterruptedException {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setMapping(
                 "text",
@@ -1993,6 +2028,7 @@ public void testHighlightNoMatchSize() throws IOException {
         String text = "I am pretty long so some of me should get cut off. Second sentence";
         index("test", "type1", "1", "text", text);
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // When you don't set noMatchSize you don't get any results if there isn't anything to highlight.
         HighlightBuilder.Field field = new HighlightBuilder.Field("text").fragmentSize(21).numOfFragments(1).highlighterType("plain");
@@ -2091,7 +2127,11 @@ public void testHighlightNoMatchSize() throws IOException {
         assertNotHighlighted(response, 0, "text");
     }
 
-    public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
+    public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException, InterruptedException {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setMapping(
                 "text",
@@ -2104,6 +2144,7 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
         String text2 = "I am short";
         index("test", "type1", "1", "text", new String[] { text1, text2 });
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // The no match fragment should come from the first value of a multi-valued field
         HighlightBuilder.Field field = new HighlightBuilder.Field("text").fragmentSize(21)
@@ -2186,7 +2227,11 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
         assertNotHighlighted(response, 0, "text");
     }
 
-    public void testHighlightNoMatchSizeNumberOfFragments() throws IOException {
+    public void testHighlightNoMatchSizeNumberOfFragments() throws IOException, InterruptedException {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setMapping(
                 "text",
@@ -2200,6 +2245,7 @@ public void testHighlightNoMatchSizeNumberOfFragments() throws IOException {
         String text3 = "This is the fifth sentence";
         index("test", "type1", "1", "text", new String[] { text1, text2, text3 });
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // The no match fragment should come from the first value of a multi-valued field
         HighlightBuilder.Field field = new HighlightBuilder.Field("text").fragmentSize(1)
@@ -2243,6 +2289,7 @@ public void testPostingsHighlighter() throws Exception {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy quick dog")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(termQuery("field1", "test"))
@@ -2320,6 +2367,7 @@ public void testPostingsHighlighterMultipleFields() throws Exception {
             "The slow brown fox. Second sentence."
         );
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchQuery("field1", "fox"))
@@ -2344,6 +2392,7 @@ public void testPostingsHighlighterNumberOfFragments() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(termQuery("field1", "fox"))
@@ -2376,6 +2425,7 @@ public void testPostingsHighlighterNumberOfFragments() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         source = searchSource().query(termQuery("field1", "fox"))
             .highlighter(highlight().field(new Field("field1").numOfFragments(0).preTags("").postTags("")));
@@ -2412,7 +2462,7 @@ public void testPostingsHighlighterNumberOfFragments() throws Exception {
         }
     }
 
-    public void testMultiMatchQueryHighlight() throws IOException {
+    public void testMultiMatchQueryHighlight() throws IOException, InterruptedException {
         XContentBuilder mapping = XContentFactory.jsonBuilder()
             .startObject()
             .startObject("properties")
@@ -2434,6 +2484,7 @@ public void testMultiMatchQueryHighlight() throws IOException {
             .setSource("field1", "The quick brown fox jumps over", "field2", "The quick brown fox jumps over")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         final int iters = scaledRandomIntBetween(20, 30);
         for (int i = 0; i < iters; i++) {
             String highlighterType = rarely() ? null : RandomPicks.randomFrom(random(), ALL_TYPES);
@@ -2479,6 +2530,7 @@ public void testPostingsHighlighterOrderByScore() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(termQuery("field1", "sentence"))
@@ -2565,6 +2617,7 @@ public void testPostingsHighlighterMultiMapperWithStore() throws Exception {
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("title", "this is a test . Second sentence.").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // simple search on body with standard analyzer with a simple field query
         SearchResponse searchResponse = client().prepareSearch()
@@ -2623,6 +2676,7 @@ public void testPostingsHighlighterMultiMapperFromSource() throws Exception {
 
         client().prepareIndex("test").setId("1").setSource("title", "this is a test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // simple search on body with standard analyzer with a simple field query
         SearchResponse searchResponse = client().prepareSearch()
@@ -2672,13 +2726,14 @@ public void testPostingsHighlighterShouldFailIfNoOffsets() throws Exception {
         assertNoFailures(search);
     }
 
-    public void testPostingsHighlighterBoostingQuery() throws IOException {
+    public void testPostingsHighlighterBoostingQuery() throws IOException, InterruptedException {
         assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping()));
         ensureGreen();
         client().prepareIndex("test")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(
@@ -2689,7 +2744,7 @@ public void testPostingsHighlighterBoostingQuery() throws IOException {
         assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick brown fox jumps over the lazy dog! Second sentence."));
     }
 
-    public void testPostingsHighlighterCommonTermsQuery() throws IOException {
+    public void testPostingsHighlighterCommonTermsQuery() throws IOException, InterruptedException {
         assertAcked(prepareCreate("test").setMapping(type1PostingsffsetsMapping()));
         ensureGreen();
 
@@ -2697,6 +2752,7 @@ public void testPostingsHighlighterCommonTermsQuery() throws IOException {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
@@ -2738,6 +2794,7 @@ public void testPostingsHighlighterPrefixQuery() throws Exception {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         logger.info("--> highlighting and searching on field2");
 
         SearchSourceBuilder source = searchSource().query(prefixQuery("field2", "qui")).highlighter(highlight().field("field2"));
@@ -2760,6 +2817,7 @@ public void testPostingsHighlighterFuzzyQuery() throws Exception {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field2");
         SearchSourceBuilder source = searchSource().query(fuzzyQuery("field2", "quck")).highlighter(highlight().field("field2"));
@@ -2783,6 +2841,7 @@ public void testPostingsHighlighterRegexpQuery() throws Exception {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field2");
         SearchSourceBuilder source = searchSource().query(regexpQuery("field2", "qu[a-l]+k")).highlighter(highlight().field("field2"));
@@ -2806,6 +2865,7 @@ public void testPostingsHighlighterWildcardQuery() throws Exception {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field2");
         SearchSourceBuilder source = searchSource().query(wildcardQuery("field2", "qui*")).highlighter(highlight().field("field2"));
@@ -2840,6 +2900,7 @@ public void testPostingsHighlighterTermRangeQuery() throws Exception {
 
         client().prepareIndex("test").setSource("field1", "this is a test", "field2", "aaab").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field2");
         SearchSourceBuilder source = searchSource().query(rangeQuery("field2").gte("aaaa").lt("zzzz"))
@@ -2857,6 +2918,7 @@ public void testPostingsHighlighterQueryString() throws Exception {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field2");
         SearchSourceBuilder source = searchSource().query(queryStringQuery("qui*").defaultField("field2"))
@@ -2878,6 +2940,7 @@ public void testPostingsHighlighterRegexpQueryWithinConstantScoreQuery() throws
 
         client().prepareIndex("test").setSource("field1", "The photography word will get highlighted").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(constantScoreQuery(regexpQuery("field1", "pho[a-z]+")))
@@ -2892,6 +2955,7 @@ public void testPostingsHighlighterMultiTermQueryMultipleLevels() throws Excepti
 
         client().prepareIndex("test").setSource("field1", "The photography word will get highlighted").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(
@@ -2909,6 +2973,7 @@ public void testPostingsHighlighterPrefixQueryWithinBooleanQuery() throws Except
 
         client().prepareIndex("test").setSource("field1", "The photography word will get highlighted").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(
@@ -2924,6 +2989,7 @@ public void testPostingsHighlighterQueryStringWithinFilteredQuery() throws Excep
 
         client().prepareIndex("test").setSource("field1", "The photography word will get highlighted").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> highlighting and searching on field1");
         SearchSourceBuilder source = searchSource().query(
@@ -3028,7 +3094,7 @@ public void testFastVectorHighlighterPhraseBoost() throws Exception {
      * because it doesn't support the concept of terms having a different weight based on position.
      * @param highlighterType highlighter to test
      */
-    private void phraseBoostTestCase(String highlighterType) {
+    private void phraseBoostTestCase(String highlighterType) throws InterruptedException {
         ensureGreen();
         StringBuilder text = new StringBuilder();
         text.append("words words junk junk junk junk junk junk junk junk highlight junk junk junk junk together junk\n");
@@ -3041,6 +3107,7 @@ private void phraseBoostTestCase(String highlighterType) {
         }
         index("test", "type1", "1", "field1", text.toString());
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // Match queries
         phraseBoostTestCaseForClauses(
@@ -3109,7 +3176,7 @@ private 
                              > void phraseBoostTestCaseForClauses(
         assertHighlight(response, 0, "field1", 0, 1, highlightedMatcher);
     }
 
-    public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOException {
+    public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOException, InterruptedException {
         // check that we do not get an exception for geo_point fields in case someone tries to highlight
         // it accidentially with a wildcard
         // see https://github.com/elastic/elasticsearch/issues/17537
@@ -3133,6 +3200,7 @@ public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOExcepti
             .setSource(jsonBuilder().startObject().field("text", "Arbitrary text field which will should not cause a failure").endObject())
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         String highlighterType = randomFrom(ALL_TYPES);
         QueryBuilder query = QueryBuilders.boolQuery()
             .should(
@@ -3150,7 +3218,7 @@ public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOExcepti
         assertThat(search.getHits().getAt(0).getHighlightFields().get("text").fragments().length, equalTo(1));
     }
 
-    public void testGeoFieldHighlightingWhenQueryGetsRewritten() throws IOException {
+    public void testGeoFieldHighlightingWhenQueryGetsRewritten() throws IOException, InterruptedException {
         // same as above but in this example the query gets rewritten during highlighting
         // see https://github.com/elastic/elasticsearch/issues/17537#issuecomment-244939633
         XContentBuilder mappings = jsonBuilder();
@@ -3177,6 +3245,7 @@ public void testGeoFieldHighlightingWhenQueryGetsRewritten() throws IOException
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         QueryBuilder query = QueryBuilders.functionScoreQuery(
             QueryBuilders.boolQuery()
@@ -3192,7 +3261,7 @@ public void testGeoFieldHighlightingWhenQueryGetsRewritten() throws IOException
         assertThat(search.getHits().getTotalHits().value, equalTo(1L));
     }
 
-    public void testKeywordFieldHighlighting() throws IOException {
+    public void testKeywordFieldHighlighting() throws IOException, InterruptedException {
         // check that keyword highlighting works
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
@@ -3205,6 +3274,7 @@ public void testKeywordFieldHighlighting() throws IOException {
             .setSource(jsonBuilder().startObject().field("keyword_field", "some text").endObject())
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         SearchResponse search = client().prepareSearch()
             .setSource(
                 new SearchSourceBuilder().query(QueryBuilders.matchQuery("keyword_field", "some text"))
@@ -3238,6 +3308,7 @@ public void testCopyToFields() throws Exception {
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse response = client().prepareSearch()
             .setQuery(matchQuery("foo_copy", "brown"))
             .highlighter(new HighlightBuilder().field(new Field("foo_copy")))
@@ -3287,7 +3358,7 @@ public void testACopyFieldWithNestedQuery() throws Exception {
             )
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
-
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(nestedQuery("foo", matchQuery("foo.text", "brown cow"), ScoreMode.None))
             .highlighter(new HighlightBuilder().field(new Field("foo_text").highlighterType("fvh")).requireFieldMatch(false))
@@ -3305,6 +3376,7 @@ public void testFunctionScoreQueryHighlight() throws Exception {
             .setSource(jsonBuilder().startObject().field("text", "brown").endObject())
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro")))
@@ -3322,6 +3394,7 @@ public void testFiltersFunctionScoreQueryHighlight() throws Exception {
             .setSource(jsonBuilder().startObject().field("text", "brown").field("enable", "yes").endObject())
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("test");
         FunctionScoreQueryBuilder.FilterFunctionBuilder filterBuilder = new FunctionScoreQueryBuilder.FilterFunctionBuilder(
             QueryBuilders.termQuery("enable", "yes"),
             new RandomScoreFunctionBuilder()
@@ -3420,6 +3493,7 @@ public void testWithNestedQuery() throws Exception {
             )
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("test");
 
         for (String type : new String[] { "unified", "plain" }) {
             SearchResponse searchResponse = client().prepareSearch()
@@ -3477,6 +3551,7 @@ public void testWithNormalizer() throws Exception {
             .setSource("keyword", "Hello World")
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("test");
 
         for (String highlighterType : new String[] { "unified", "plain" }) {
             SearchResponse searchResponse = client().prepareSearch()
@@ -3499,6 +3574,7 @@ public void testDisableHighlightIdField() throws Exception {
             .setSource("keyword", "Hello World")
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("test");
 
         for (String highlighterType : new String[] { "plain", "unified" }) {
             SearchResponse searchResponse = client().prepareSearch()
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index ad27d9834f159..0c6c81103922f 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -1666,6 +1666,11 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
             }
         }
         assertThat(actualErrors, emptyIterable());
+
+        if (dummyDocuments) {
+            bogusIds.addAll(indexRandomForMultipleSlices(indicesArray));
+        }
+
         if (!bogusIds.isEmpty()) {
             // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
             for (List doc : bogusIds) {
@@ -1683,6 +1688,52 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
         }
     }
 
+    /*
+    * This method ingests bogus documents for the given indices such that multiple slices
+    * are formed. This is useful for testing with the concurrent search use-case as it creates
+    * multiple slices based on segment count.
+    * @param indices         the indices in which bogus documents should be ingested
+    * */
+    protected Set> indexRandomForMultipleSlices(String... indices) throws InterruptedException {
+        Set> bogusIds = new HashSet<>();
+        int refreshCount = randomIntBetween(2, 3);
+        for (String index : indices) {
+            int numDocs = getNumShards(index).totalNumShards * randomIntBetween(2, 10);
+            while (refreshCount-- > 0) {
+                final CopyOnWriteArrayList> errors = new CopyOnWriteArrayList<>();
+                List inFlightAsyncOperations = new ArrayList<>();
+                for (int i = 0; i < numDocs; i++) {
+                    String id = "bogus_doc_" + randomRealisticUnicodeOfLength(between(1, 10)) + dummmyDocIdGenerator.incrementAndGet();
+                    IndexRequestBuilder indexRequestBuilder = client().prepareIndex()
+                        .setIndex(index)
+                        .setId(id)
+                        .setSource("{}", MediaTypeRegistry.JSON)
+                        .setRouting(id);
+                    indexRequestBuilder.execute(
+                        new PayloadLatchedActionListener<>(indexRequestBuilder, newLatch(inFlightAsyncOperations), errors)
+                    );
+                    bogusIds.add(Arrays.asList(index, id));
+                }
+                for (CountDownLatch operation : inFlightAsyncOperations) {
+                    operation.await();
+                }
+                final List actualErrors = new ArrayList<>();
+                for (Tuple tuple : errors) {
+                    Throwable t = ExceptionsHelper.unwrapCause(tuple.v2());
+                    if (t instanceof OpenSearchRejectedExecutionException) {
+                        logger.debug("Error indexing doc: " + t.getMessage() + ", reindexing.");
+                        tuple.v1().execute().actionGet(); // re-index if rejected
+                    } else {
+                        actualErrors.add(tuple.v2());
+                    }
+                }
+                assertThat(actualErrors, emptyIterable());
+                refresh(index);
+            }
+        }
+        return bogusIds;
+    }
+
     private final AtomicInteger dummmyDocIdGenerator = new AtomicInteger();
 
     /** Disables an index block for the specified index */
diff --git a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java
index 636064d8e4f9d..f8813a8c5afa9 100644
--- a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java
@@ -13,6 +13,8 @@
 import org.junit.After;
 import org.junit.Before;
 
+import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
+
 /**
  * Base class for running the tests with parameterization of the dynamic settings
  * For any class that wants to use parameterization, use @ParametersFactory to generate
@@ -44,4 +46,10 @@ public void afterTests() {
         dynamicSettings.keySet().forEach(settingsToUnset::putNull);
         client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settingsToUnset).get();
     }
+
+    public void indexRandomForConcurrentSearch(String... indices) throws InterruptedException {
+        if (dynamicSettings.get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey()).equals("true")) {
+            indexRandomForMultipleSlices(indices);
+        }
+    }
 }

From 45f7be1c603d13a1ce48e5da449ae4610ba04e52 Mon Sep 17 00:00:00 2001
From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com>
Date: Sat, 28 Oct 2023 15:43:14 +0530
Subject: [PATCH 248/388] =?UTF-8?q?Ensure=20that=20segments=20are=20upload?=
 =?UTF-8?q?=20to=20remote=20store=20in=20case=20of=20local=20and=20?=
 =?UTF-8?q?=E2=80=A6=20(#10948)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---------

Signed-off-by: Gaurav Bafna 
---
 .../indices/create/RemoteCloneIndexIT.java    | 133 +++++
 .../indices/create/RemoteShrinkIndexIT.java   | 545 ++++++++++++++++++
 .../indices/create/RemoteSplitIndexIT.java    | 506 ++++++++++++++++
 .../remotestore/RemoteRestoreSnapshotIT.java  |  43 +-
 .../RemoteStoreBaseIntegTestCase.java         |   2 +-
 .../opensearch/index/shard/IndexShard.java    |  24 +
 .../shard/RemoteStoreRefreshListener.java     |  22 +-
 .../opensearch/index/shard/StoreRecovery.java |  21 +
 .../index/shard/IndexShardTests.java          |   1 +
 9 files changed, 1278 insertions(+), 19 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteShrinkIndexIT.java
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java

diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
new file mode 100644
index 0000000000000..a081110e6c5a1
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
@@ -0,0 +1,133 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.admin.indices.create;
+
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import org.opensearch.Version;
+import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.opensearch.action.admin.indices.shrink.ResizeType;
+import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.index.query.TermsQueryBuilder;
+import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
+import org.opensearch.test.VersionUtils;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RemoteCloneIndexIT extends RemoteStoreBaseIntegTestCase {
+
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        return false;
+    }
+
+    public void testCreateCloneIndex() {
+        Version version = VersionUtils.randomIndexCompatibleVersion(random());
+        int numPrimaryShards = randomIntBetween(1, 5);
+        prepareCreate("source").setSettings(
+            Settings.builder().put(indexSettings()).put("number_of_shards", numPrimaryShards).put("index.version.created", version)
+        ).get();
+        final int docs = randomIntBetween(0, 128);
+        for (int i = 0; i < docs; i++) {
+            client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+        }
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.blocks.write", true)).get();
+        ensureGreen();
+
+        final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get();
+
+        // disable rebalancing to be able to capture the right stats. balancing can move the target primary
+        // making it hard to pin point the source shards.
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none"))
+            .get();
+        try {
+            assertAcked(
+                client().admin()
+                    .indices()
+                    .prepareResizeIndex("source", "target")
+                    .setResizeType(ResizeType.CLONE)
+                    .setSettings(Settings.builder().put("index.number_of_replicas", 0).putNull("index.blocks.write").build())
+                    .get()
+            );
+            ensureGreen();
+
+            final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get();
+            assertThat(targetStats.getIndex("target").getIndexShards().keySet().size(), equalTo(numPrimaryShards));
+
+            final int size = docs > 0 ? 2 * docs : 1;
+            assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
+
+            for (int i = docs; i < 2 * docs; i++) {
+                client().prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+            }
+            flushAndRefresh();
+            assertHitCount(
+                client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(),
+                2 * docs
+            );
+            assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
+            GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
+            assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
+        } finally {
+            // clean up
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(
+                    Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                )
+                .get();
+        }
+
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteShrinkIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteShrinkIndexIT.java
new file mode 100644
index 0000000000000..282eb9c6ad95e
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteShrinkIndexIT.java
@@ -0,0 +1,545 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.admin.indices.create;
+
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.util.Constants;
+import org.opensearch.Version;
+import org.opensearch.action.admin.cluster.reroute.ClusterRerouteResponse;
+import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
+import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
+import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.opensearch.action.admin.indices.stats.CommonStats;
+import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.opensearch.action.admin.indices.stats.ShardStats;
+import org.opensearch.action.index.IndexRequest;
+import org.opensearch.action.support.ActiveShardCount;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.ClusterInfoService;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.InternalClusterInfoService;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.routing.Murmur3HashFunction;
+import org.opensearch.cluster.routing.RoutingTable;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.UnassignedInfo;
+import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.opensearch.common.Priority;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.index.IndexModule;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.engine.SegmentsStats;
+import org.opensearch.index.query.TermsQueryBuilder;
+import org.opensearch.index.seqno.SeqNoStats;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
+import org.opensearch.test.VersionUtils;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.stream.IntStream;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+public class RemoteShrinkIndexIT extends RemoteStoreBaseIntegTestCase {
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        return false;
+    }
+
+    public Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false)
+            .put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .build();
+    }
+
+    public void testCreateShrinkIndexToN() {
+
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/34080", Constants.WINDOWS);
+
+        int[][] possibleShardSplits = new int[][] { { 8, 4, 2 }, { 9, 3, 1 }, { 4, 2, 1 }, { 15, 5, 1 } };
+        int[] shardSplits = randomFrom(possibleShardSplits);
+        assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]);
+        assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source")
+                .setId(Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON)
+                .get();
+        }
+        final Map dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(new DiscoveryNode[0]);
+        String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode).put("index.blocks.write", true))
+            .get();
+        ensureGreen();
+        // now merge source into a 4 shard index
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareResizeIndex("source", "first_shrink")
+                .setSettings(
+                    Settings.builder()
+                        .put("index.number_of_replicas", 0)
+                        .put("index.number_of_shards", shardSplits[1])
+                        .putNull("index.blocks.write")
+                        .build()
+                )
+                .get()
+        );
+        ensureGreen();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("first_shrink")
+                .setId(Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON)
+                .get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        // relocate all shards to one node such that we can merge it.
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("first_shrink")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode).put("index.blocks.write", true))
+            .get();
+        ensureGreen();
+        // now merge source into a 2 shard index
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareResizeIndex("first_shrink", "second_shrink")
+                .setSettings(
+                    Settings.builder()
+                        .put("index.number_of_replicas", 0)
+                        .put("index.number_of_shards", shardSplits[2])
+                        .putNull("index.blocks.write")
+                        .putNull("index.routing.allocation.require._name")
+                        .build()
+                )
+                .get()
+        );
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        // let it be allocated anywhere and bump replicas
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("second_shrink")
+            .setSettings(Settings.builder().putNull("index.routing.allocation.include._id").put("index.number_of_replicas", 0))
+            .get();
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("second_shrink")
+                .setId(Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON)
+                .get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
+
+    public void testShrinkIndexPrimaryTerm() throws Exception {
+        int numberOfShards = randomIntBetween(2, 20);
+        int numberOfTargetShards = randomValueOtherThanMany(n -> numberOfShards % n != 0, () -> randomIntBetween(1, numberOfShards - 1));
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", numberOfShards)).get();
+
+        final Map dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
+        assertThat(dataNodes.size(), greaterThanOrEqualTo(2));
+        final DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(new DiscoveryNode[0]);
+        final String mergeNode = discoveryNodes[0].getName();
+        // This needs more than the default timeout if a large number of shards were created.
+        ensureGreen(TimeValue.timeValueSeconds(120));
+
+        // fail random primary shards to force primary terms to increase
+        final Index source = resolveIndex("source");
+        final int iterations = scaledRandomIntBetween(0, 16);
+        for (int i = 0; i < iterations; i++) {
+            final String node = randomSubsetOf(1, internalCluster().nodesInclude("source")).get(0);
+            final IndicesService indexServices = internalCluster().getInstance(IndicesService.class, node);
+            final IndexService indexShards = indexServices.indexServiceSafe(source);
+            for (final Integer shardId : indexShards.shardIds()) {
+                final IndexShard shard = indexShards.getShard(shardId);
+                if (shard.routingEntry().primary() && randomBoolean()) {
+                    disableAllocation("source");
+                    shard.failShard("test", new Exception("test"));
+                    // this can not succeed until the shard is failed and a replica is promoted
+                    int id = 0;
+                    while (true) {
+                        // find an ID that routes to the right shard, we will only index to the shard that saw a primary failure
+                        final String s = Integer.toString(id);
+                        final int hash = Math.floorMod(Murmur3HashFunction.hash(s), numberOfShards);
+                        if (hash == shardId) {
+                            final IndexRequest request = new IndexRequest("source").id(s)
+                                .source("{ \"f\": \"" + s + "\"}", MediaTypeRegistry.JSON);
+                            client().index(request).get();
+                            break;
+                        } else {
+                            id++;
+                        }
+                    }
+                    enableAllocation("source");
+                    ensureGreen();
+                }
+            }
+        }
+
+        // relocate all shards to one node such that we can merge it.
+        final Settings.Builder prepareShrinkSettings = Settings.builder()
+            .put("index.routing.allocation.require._name", mergeNode)
+            .put("index.blocks.write", true);
+        client().admin().indices().prepareUpdateSettings("source").setSettings(prepareShrinkSettings).get();
+        ensureGreen(TimeValue.timeValueSeconds(120)); // needs more than the default to relocate many shards
+
+        final IndexMetadata indexMetadata = indexMetadata(client(), "source");
+        final long beforeShrinkPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetadata::primaryTerm).max().getAsLong();
+
+        // now merge source into target
+        final Settings shrinkSettings = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", numberOfTargetShards)
+            .build();
+        assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setSettings(shrinkSettings).get());
+
+        ensureGreen(TimeValue.timeValueSeconds(120));
+
+        final IndexMetadata afterShrinkIndexMetadata = indexMetadata(client(), "target");
+        for (int shardId = 0; shardId < numberOfTargetShards; shardId++) {
+            assertThat(afterShrinkIndexMetadata.primaryTerm(shardId), equalTo(beforeShrinkPrimaryTerm + 1));
+        }
+    }
+
+    private static IndexMetadata indexMetadata(final Client client, final String index) {
+        final ClusterStateResponse clusterStateResponse = client.admin().cluster().state(new ClusterStateRequest()).actionGet();
+        return clusterStateResponse.getState().metadata().index(index);
+    }
+
+    public void testCreateShrinkIndex() {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        Version version = VersionUtils.randomVersion(random());
+        prepareCreate("source").setSettings(
+            Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7)).put("index.version.created", version)
+        ).get();
+        final int docs = randomIntBetween(0, 128);
+        for (int i = 0; i < docs; i++) {
+            client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+        }
+        final Map dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(new DiscoveryNode[0]);
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("source")
+            .setSettings(
+                Settings.builder()
+                    .put("index.routing.allocation.require._name", discoveryNodes[0].getName())
+                    .put("index.blocks.write", true)
+            )
+            .get();
+        ensureGreen();
+
+        final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get();
+
+        // disable rebalancing to be able to capture the right stats. balancing can move the target primary
+        // making it hard to pin point the source shards.
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none"))
+            .get();
+
+        // now merge source into a single shard index
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareResizeIndex("source", "target")
+                .setSettings(
+                    Settings.builder()
+                        .put("index.number_of_replicas", 0)
+                        .putNull("index.blocks.write")
+                        .putNull("index.routing.allocation.require._name")
+                        .build()
+                )
+                .get()
+        );
+        ensureGreen();
+
+        // resolve true merge node - this is not always the node we required as all shards may be on another node
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId());
+        logger.info("merge node {}", mergeNode);
+
+        final long maxSeqNo = Arrays.stream(sourceStats.getShards())
+            .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId()))
+            .map(ShardStats::getSeqNoStats)
+            .mapToLong(SeqNoStats::getMaxSeqNo)
+            .max()
+            .getAsLong();
+        final long maxUnsafeAutoIdTimestamp = Arrays.stream(sourceStats.getShards())
+            .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId()))
+            .map(ShardStats::getStats)
+            .map(CommonStats::getSegments)
+            .mapToLong(SegmentsStats::getMaxUnsafeAutoIdTimestamp)
+            .max()
+            .getAsLong();
+
+        final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get();
+        for (final ShardStats shardStats : targetStats.getShards()) {
+            final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
+            final ShardRouting shardRouting = shardStats.getShardRouting();
+            assertThat("failed on " + shardRouting, seqNoStats.getMaxSeqNo(), equalTo(maxSeqNo));
+            assertThat("failed on " + shardRouting, seqNoStats.getLocalCheckpoint(), equalTo(maxSeqNo));
+            assertThat(
+                "failed on " + shardRouting,
+                shardStats.getStats().getSegments().getMaxUnsafeAutoIdTimestamp(),
+                equalTo(maxUnsafeAutoIdTimestamp)
+            );
+        }
+
+        final int size = docs > 0 ? 2 * docs : 1;
+        assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
+
+        for (int i = docs; i < 2 * docs; i++) {
+            client().prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 2 * docs);
+        assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
+        GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
+        assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
+
+        // clean up
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+            )
+            .get();
+    }
+
+    /**
+     * Tests that we can manually recover from a failed allocation due to shards being moved away etc.
+     */
+    public void testCreateShrinkIndexFails() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(
+            Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7)).put("number_of_replicas", 0)
+        ).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+        }
+        final Map dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(new DiscoveryNode[0]);
+        String spareNode = discoveryNodes[0].getName();
+        String mergeNode = discoveryNodes[1].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode).put("index.blocks.write", true))
+            .get();
+        ensureGreen();
+
+        // now merge source into a single shard index
+        client().admin()
+            .indices()
+            .prepareResizeIndex("source", "target")
+            .setWaitForActiveShards(ActiveShardCount.NONE)
+            .setSettings(
+                Settings.builder()
+                    .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up
+                    .put("index.number_of_replicas", 0)
+                    .put("index.allocation.max_retries", 1)
+                    .build()
+            )
+            .get();
+        client().admin().cluster().prepareHealth("target").setWaitForEvents(Priority.LANGUID).get();
+
+        // now we move all shards away from the merge node
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", spareNode).put("index.blocks.write", true))
+            .get();
+        ensureGreen("source");
+
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("target") // erase the forcefully fuckup!
+            .setSettings(Settings.builder().putNull("index.routing.allocation.exclude._name"))
+            .get();
+        // wait until it fails
+        assertBusy(() -> {
+            ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
+            RoutingTable routingTables = clusterStateResponse.getState().routingTable();
+            assertTrue(routingTables.index("target").shard(0).getShards().get(0).unassigned());
+            assertEquals(
+                UnassignedInfo.Reason.ALLOCATION_FAILED,
+                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getReason()
+            );
+            assertEquals(1, routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getNumFailedAllocations());
+        });
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("source") // now relocate them all to the right node
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode))
+            .get();
+        ensureGreen("source");
+
+        final InternalClusterInfoService infoService = (InternalClusterInfoService) internalCluster().getInstance(
+            ClusterInfoService.class,
+            internalCluster().getClusterManagerName()
+        );
+        infoService.refresh();
+        // kick off a retry and wait until it's done!
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        long expectedShardSize = clusterRerouteResponse.getState()
+            .routingTable()
+            .index("target")
+            .shard(0)
+            .getShards()
+            .get(0)
+            .getExpectedShardSize();
+        // we support the expected shard size in the allocator to sum up over the source index shards
+        assertTrue("expected shard size must be set but wasn't: " + expectedShardSize, expectedShardSize > 0);
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
+
+    public void testCreateShrinkWithIndexSort() throws Exception {
+        SortField expectedSortField = new SortedSetSortField("id", true, SortedSetSelector.Type.MAX);
+        expectedSortField.setMissingValue(SortedSetSortField.STRING_FIRST);
+        Sort expectedIndexSort = new Sort(expectedSortField);
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(
+            Settings.builder()
+                .put(indexSettings())
+                .put("sort.field", "id")
+                .put("sort.order", "desc")
+                .put("number_of_shards", 8)
+                .put("number_of_replicas", 0)
+        ).setMapping("id", "type=keyword,doc_values=true").get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source")
+                .setId(Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"id\" : " + i + "}", MediaTypeRegistry.JSON)
+                .get();
+        }
+        final Map dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(new DiscoveryNode[0]);
+        String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+
+        flushAndRefresh();
+        assertSortedSegments("source", expectedIndexSort);
+
+        // relocate all shards to one node such that we can merge it.
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode).put("index.blocks.write", true))
+            .get();
+        ensureGreen();
+
+        // check that index sort cannot be set on the target index
+        IllegalArgumentException exc = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin()
+                .indices()
+                .prepareResizeIndex("source", "target")
+                .setSettings(
+                    Settings.builder()
+                        .put("index.number_of_replicas", 0)
+                        .put("index.number_of_shards", "2")
+                        .put("index.sort.field", "foo")
+                        .build()
+                )
+                .get()
+        );
+        assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index"));
+
+        // check that the index sort order of `source` is correctly applied to the `target`
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareResizeIndex("source", "target")
+                .setSettings(
+                    Settings.builder()
+                        .put("index.number_of_replicas", 0)
+                        .put("index.number_of_shards", "2")
+                        .putNull("index.blocks.write")
+                        .build()
+                )
+                .get()
+        );
+        ensureGreen();
+        flushAndRefresh();
+        GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("target").execute().actionGet();
+        assertEquals(settingsResponse.getSetting("target", "index.sort.field"), "id");
+        assertEquals(settingsResponse.getSetting("target", "index.sort.order"), "desc");
+        assertSortedSegments("target", expectedIndexSort);
+
+        // ... and that the index sort is also applied to updates
+        for (int i = 20; i < 40; i++) {
+            client().prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+        }
+        flushAndRefresh();
+        assertSortedSegments("target", expectedIndexSort);
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java
new file mode 100644
index 0000000000000..dd4252d24f314
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java
@@ -0,0 +1,506 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.action.admin.indices.create;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.util.Constants;
+import org.opensearch.Version;
+import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
+import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
+import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.opensearch.action.admin.indices.shrink.ResizeType;
+import org.opensearch.action.admin.indices.stats.CommonStats;
+import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.opensearch.action.admin.indices.stats.ShardStats;
+import org.opensearch.action.get.GetResponse;
+import org.opensearch.action.index.IndexRequest;
+import org.opensearch.action.index.IndexRequestBuilder;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.MetadataCreateIndexService;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.routing.Murmur3HashFunction;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.index.IndexModule;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.engine.SegmentsStats;
+import org.opensearch.index.query.TermsQueryBuilder;
+import org.opensearch.index.seqno.SeqNoStats;
+import org.opensearch.index.shard.IndexShard;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
+import org.opensearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.BiFunction;
+import java.util.stream.IntStream;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.opensearch.index.query.QueryBuilders.nestedQuery;
+import static org.opensearch.index.query.QueryBuilders.termQuery;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RemoteSplitIndexIT extends RemoteStoreBaseIntegTestCase {
+
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        return false;
+    }
+
+    public Settings indexSettings() {
+        return Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false)
+            .put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .build();
+    }
+
+    public void testCreateSplitIndexToN() throws IOException {
+        int[][] possibleShardSplits = new int[][] { { 2, 4, 8 }, { 3, 6, 12 }, { 1, 2, 4 } };
+        int[] shardSplits = randomFrom(possibleShardSplits);
+        splitToN(shardSplits[0], shardSplits[1], shardSplits[2]);
+    }
+
+    public void testSplitFromOneToN() {
+
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/34080", Constants.WINDOWS);
+
+        splitToN(1, 5, 10);
+        client().admin().indices().prepareDelete("*").get();
+        int randomSplit = randomIntBetween(2, 6);
+        splitToN(1, randomSplit, randomSplit * 2);
+    }
+
+    private void splitToN(int sourceShards, int firstSplitShards, int secondSplitShards) {
+
+        assertEquals(sourceShards, (sourceShards * firstSplitShards) / firstSplitShards);
+        assertEquals(firstSplitShards, (firstSplitShards * secondSplitShards) / secondSplitShards);
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        final boolean useRouting = randomBoolean();
+        final boolean useNested = randomBoolean();
+        final boolean useMixedRouting = useRouting ? randomBoolean() : false;
+        CreateIndexRequestBuilder createInitialIndex = prepareCreate("source");
+        Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", sourceShards);
+        final boolean useRoutingPartition;
+        if (randomBoolean()) {
+            // randomly set the value manually
+            int routingShards = secondSplitShards * randomIntBetween(1, 10);
+            settings.put("index.number_of_routing_shards", routingShards);
+            useRoutingPartition = false;
+        } else {
+            useRoutingPartition = randomBoolean();
+        }
+        if (useRouting && useMixedRouting == false && useRoutingPartition) {
+            int numRoutingShards = MetadataCreateIndexService.calculateNumRoutingShards(secondSplitShards, Version.CURRENT) - 1;
+            settings.put("index.routing_partition_size", randomIntBetween(1, numRoutingShards));
+            if (useNested) {
+                createInitialIndex.setMapping("_routing", "required=true", "nested1", "type=nested");
+            } else {
+                createInitialIndex.setMapping("_routing", "required=true");
+            }
+        } else if (useNested) {
+            createInitialIndex.setMapping("nested1", "type=nested");
+        }
+        logger.info("use routing {} use mixed routing {} use nested {}", useRouting, useMixedRouting, useNested);
+        createInitialIndex.setSettings(settings).get();
+
+        int numDocs = randomIntBetween(10, 50);
+        String[] routingValue = new String[numDocs];
+
+        BiFunction indexFunc = (index, id) -> {
+            try {
+                return client().prepareIndex(index)
+                    .setId(Integer.toString(id))
+                    .setSource(
+                        jsonBuilder().startObject()
+                            .field("foo", "bar")
+                            .field("i", id)
+                            .startArray("nested1")
+                            .startObject()
+                            .field("n_field1", "n_value1_1")
+                            .field("n_field2", "n_value2_1")
+                            .endObject()
+                            .startObject()
+                            .field("n_field1", "n_value1_2")
+                            .field("n_field2", "n_value2_2")
+                            .endObject()
+                            .endArray()
+                            .endObject()
+                    );
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        };
+        for (int i = 0; i < numDocs; i++) {
+            IndexRequestBuilder builder = indexFunc.apply("source", i);
+            if (useRouting) {
+                String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 10);
+                if (useMixedRouting && randomBoolean()) {
+                    routingValue[i] = null;
+                } else {
+                    routingValue[i] = routing;
+                }
+                builder.setRouting(routingValue[i]);
+            }
+            builder.get();
+        }
+
+        if (randomBoolean()) {
+            for (int i = 0; i < numDocs; i++) { // let's introduce some updates / deletes on the index
+                if (randomBoolean()) {
+                    IndexRequestBuilder builder = indexFunc.apply("source", i);
+                    if (useRouting) {
+                        builder.setRouting(routingValue[i]);
+                    }
+                    builder.get();
+                }
+            }
+        }
+
+        ensureYellow();
+        client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.blocks.write", true)).get();
+        ensureGreen();
+        Settings.Builder firstSplitSettingsBuilder = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", firstSplitShards)
+            .putNull("index.blocks.write");
+        if (sourceShards == 1 && useRoutingPartition == false && randomBoolean()) { // try to set it if we have a source index with 1 shard
+            firstSplitSettingsBuilder.put("index.number_of_routing_shards", secondSplitShards);
+        }
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareResizeIndex("source", "first_split")
+                .setResizeType(ResizeType.SPLIT)
+                .setSettings(firstSplitSettingsBuilder.build())
+                .get()
+        );
+        ensureGreen();
+        assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+
+        for (int i = 0; i < numDocs; i++) { // now update
+            IndexRequestBuilder builder = indexFunc.apply("first_split", i);
+            if (useRouting) {
+                builder.setRouting(routingValue[i]);
+            }
+            builder.get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+        for (int i = 0; i < numDocs; i++) {
+            GetResponse getResponse = client().prepareGet("first_split", Integer.toString(i)).setRouting(routingValue[i]).get();
+            assertTrue(getResponse.isExists());
+        }
+
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("first_split")
+            .setSettings(Settings.builder().put("index.blocks.write", true))
+            .get();
+        ensureGreen();
+        // now split source into a new index
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareResizeIndex("first_split", "second_split")
+                .setResizeType(ResizeType.SPLIT)
+                .setSettings(
+                    Settings.builder()
+                        .put("index.number_of_replicas", 0)
+                        .put("index.number_of_shards", secondSplitShards)
+                        .putNull("index.blocks.write")
+                        .build()
+                )
+                .get()
+        );
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+        // let it be allocated anywhere and bump replicas
+        client().admin()
+            .indices()
+            .prepareUpdateSettings("second_split")
+            .setSettings(Settings.builder().put("index.number_of_replicas", 0))
+            .get();
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+
+        for (int i = 0; i < numDocs; i++) { // now update
+            IndexRequestBuilder builder = indexFunc.apply("second_split", i);
+            if (useRouting) {
+                builder.setRouting(routingValue[i]);
+            }
+            builder.get();
+        }
+        flushAndRefresh();
+        for (int i = 0; i < numDocs; i++) {
+            GetResponse getResponse = client().prepareGet("second_split", Integer.toString(i)).setRouting(routingValue[i]).get();
+            assertTrue(getResponse.isExists());
+        }
+        assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+        assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+        if (useNested) {
+            assertNested("source", numDocs);
+            assertNested("first_split", numDocs);
+            assertNested("second_split", numDocs);
+        }
+        assertAllUniqueDocs(
+            client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(),
+            numDocs
+        );
+        assertAllUniqueDocs(
+            client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(),
+            numDocs
+        );
+        assertAllUniqueDocs(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs);
+    }
+
+    public void assertNested(String index, int numDocs) {
+        // now, do a nested query
+        SearchResponse searchResponse = client().prepareSearch(index)
+            .setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"), ScoreMode.Avg))
+            .get();
+        assertNoFailures(searchResponse);
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) numDocs));
+    }
+
+    public void assertAllUniqueDocs(SearchResponse response, int numDocs) {
+        Set ids = new HashSet<>();
+        for (int i = 0; i < response.getHits().getHits().length; i++) {
+            String id = response.getHits().getHits()[i].getId();
+            assertTrue("found ID " + id + " more than once", ids.add(id));
+        }
+        assertEquals(numDocs, ids.size());
+    }
+
+    public void testSplitIndexPrimaryTerm() throws Exception {
+        int numberOfTargetShards = randomIntBetween(2, 20);
+        int numberOfShards = randomValueOtherThanMany(n -> numberOfTargetShards % n != 0, () -> between(1, numberOfTargetShards - 1));
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(
+            Settings.builder()
+                .put(indexSettings())
+                .put("number_of_shards", numberOfShards)
+                .put("index.number_of_routing_shards", numberOfTargetShards)
+        ).get();
+        ensureGreen(TimeValue.timeValueSeconds(120)); // needs more than the default to allocate many shards
+
+        // fail random primary shards to force primary terms to increase
+        final Index source = resolveIndex("source");
+        final int iterations = scaledRandomIntBetween(0, 16);
+        for (int i = 0; i < iterations; i++) {
+            final String node = randomSubsetOf(1, internalCluster().nodesInclude("source")).get(0);
+            final IndicesService indexServices = internalCluster().getInstance(IndicesService.class, node);
+            final IndexService indexShards = indexServices.indexServiceSafe(source);
+            for (final Integer shardId : indexShards.shardIds()) {
+                final IndexShard shard = indexShards.getShard(shardId);
+                if (shard.routingEntry().primary() && randomBoolean()) {
+                    disableAllocation("source");
+                    shard.failShard("test", new Exception("test"));
+                    // this can not succeed until the shard is failed and a replica is promoted
+                    int id = 0;
+                    while (true) {
+                        // find an ID that routes to the right shard, we will only index to the shard that saw a primary failure
+                        final String s = Integer.toString(id);
+                        final int hash = Math.floorMod(Murmur3HashFunction.hash(s), numberOfShards);
+                        if (hash == shardId) {
+                            final IndexRequest request = new IndexRequest("source").id(s)
+                                .source("{ \"f\": \"" + s + "\"}", MediaTypeRegistry.JSON);
+                            client().index(request).get();
+                            break;
+                        } else {
+                            id++;
+                        }
+                    }
+                    enableAllocation("source");
+                    ensureGreen();
+                }
+            }
+        }
+
+        final Settings.Builder prepareSplitSettings = Settings.builder().put("index.blocks.write", true);
+        client().admin().indices().prepareUpdateSettings("source").setSettings(prepareSplitSettings).get();
+        ensureYellow();
+
+        final IndexMetadata indexMetadata = indexMetadata(client(), "source");
+        final long beforeSplitPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetadata::primaryTerm).max().getAsLong();
+
+        // now split source into target
+        final Settings splitSettings = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", numberOfTargetShards)
+            .putNull("index.blocks.write")
+            .build();
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareResizeIndex("source", "target")
+                .setResizeType(ResizeType.SPLIT)
+                .setSettings(splitSettings)
+                .get()
+        );
+
+        ensureGreen(TimeValue.timeValueSeconds(120)); // needs more than the default to relocate many shards
+
+        final IndexMetadata aftersplitIndexMetadata = indexMetadata(client(), "target");
+        for (int shardId = 0; shardId < numberOfTargetShards; shardId++) {
+            assertThat(aftersplitIndexMetadata.primaryTerm(shardId), equalTo(beforeSplitPrimaryTerm + 1));
+        }
+    }
+
+    private static IndexMetadata indexMetadata(final Client client, final String index) {
+        final ClusterStateResponse clusterStateResponse = client.admin().cluster().state(new ClusterStateRequest()).actionGet();
+        return clusterStateResponse.getState().metadata().index(index);
+    }
+
+    public void testCreateSplitIndex() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        Version version = VersionUtils.randomIndexCompatibleVersion(random());
+        prepareCreate("source").setSettings(
+            Settings.builder().put(indexSettings()).put("number_of_shards", 1).put("index.version.created", version)
+        ).get();
+        final int docs = randomIntBetween(0, 128);
+        for (int i = 0; i < docs; i++) {
+            client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+        }
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.blocks.write", true)).get();
+        ensureGreen();
+
+        final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get();
+
+        // disable rebalancing to be able to capture the right stats. balancing can move the target primary
+        // making it hard to pin point the source shards.
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none"))
+            .get();
+        try {
+            assertAcked(
+                client().admin()
+                    .indices()
+                    .prepareResizeIndex("source", "target")
+                    .setResizeType(ResizeType.SPLIT)
+                    .setSettings(
+                        Settings.builder()
+                            .put("index.number_of_replicas", 0)
+                            .put("index.number_of_shards", 2)
+                            .putNull("index.blocks.write")
+                            .build()
+                    )
+                    .get()
+            );
+            ensureGreen();
+
+            final ClusterState state = client().admin().cluster().prepareState().get().getState();
+            DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId());
+            logger.info("split node {}", mergeNode);
+
+            final long maxSeqNo = Arrays.stream(sourceStats.getShards())
+                .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId()))
+                .map(ShardStats::getSeqNoStats)
+                .mapToLong(SeqNoStats::getMaxSeqNo)
+                .max()
+                .getAsLong();
+            final long maxUnsafeAutoIdTimestamp = Arrays.stream(sourceStats.getShards())
+                .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId()))
+                .map(ShardStats::getStats)
+                .map(CommonStats::getSegments)
+                .mapToLong(SegmentsStats::getMaxUnsafeAutoIdTimestamp)
+                .max()
+                .getAsLong();
+
+            final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get();
+            for (final ShardStats shardStats : targetStats.getShards()) {
+                final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
+                final ShardRouting shardRouting = shardStats.getShardRouting();
+                assertThat("failed on " + shardRouting, seqNoStats.getMaxSeqNo(), equalTo(maxSeqNo));
+                assertThat("failed on " + shardRouting, seqNoStats.getLocalCheckpoint(), equalTo(maxSeqNo));
+                assertThat(
+                    "failed on " + shardRouting,
+                    shardStats.getStats().getSegments().getMaxUnsafeAutoIdTimestamp(),
+                    equalTo(maxUnsafeAutoIdTimestamp)
+                );
+            }
+
+            final int size = docs > 0 ? 2 * docs : 1;
+            assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
+
+            for (int i = docs; i < 2 * docs; i++) {
+                client().prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", MediaTypeRegistry.JSON).get();
+            }
+            flushAndRefresh();
+            assertHitCount(
+                client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(),
+                2 * docs
+            );
+            assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
+            GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
+            assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
+        } finally {
+            // clean up
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(
+                    Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                )
+                .get();
+        }
+
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
index 9e0b2a66467de..ad78c503a4a19 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
@@ -330,6 +330,8 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
         assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED);
         assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED);
         ensureGreen(indexName1, restoredIndexName2);
+
+        assertRemoteSegmentsAndTranslogUploaded(restoredIndexName2);
         assertDocsPresentInIndex(client, indexName1, numDocsInIndex1);
         assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2);
         // indexing some new docs and validating
@@ -355,6 +357,29 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
         assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 4);
     }
 
+    void assertRemoteSegmentsAndTranslogUploaded(String idx) throws IOException {
+        String indexUUID = client().admin().indices().prepareGetSettings(idx).get().getSetting(idx, IndexMetadata.SETTING_INDEX_UUID);
+
+        Path remoteTranslogMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/metadata");
+        Path remoteTranslogDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/data");
+        Path segmentMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/segments/metadata");
+        Path segmentDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/segments/data");
+
+        try (
+            Stream translogMetadata = Files.list(remoteTranslogMetadataPath);
+            Stream translogData = Files.list(remoteTranslogDataPath);
+            Stream segmentMetadata = Files.list(segmentMetadataPath);
+            Stream segmentData = Files.list(segmentDataPath);
+
+        ) {
+            assertTrue(translogData.count() > 0);
+            assertTrue(translogMetadata.count() > 0);
+            assertTrue(segmentMetadata.count() > 0);
+            assertTrue(segmentData.count() > 0);
+        }
+
+    }
+
     public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, ExecutionException, InterruptedException {
         internalCluster().startClusterManagerOnlyNode();
         internalCluster().startDataOnlyNodes(2);
@@ -395,23 +420,7 @@ public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, Exe
         ensureGreen(indexName1);
         assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
 
-        // Make sure remote translog is empty
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(indexName1)
-            .get()
-            .getSetting(indexName1, IndexMetadata.SETTING_INDEX_UUID);
-
-        Path remoteTranslogMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/metadata");
-        Path remoteTranslogDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/data");
-
-        try (
-            Stream translogMetadata = Files.list(remoteTranslogMetadataPath);
-            Stream translogData = Files.list(remoteTranslogDataPath)
-        ) {
-            assertTrue(translogData.count() > 0);
-            assertTrue(translogMetadata.count() > 0);
-        }
+        assertRemoteSegmentsAndTranslogUploaded(indexName1);
 
         // Clear the local data before stopping the node. This will make sure that remote translog is empty.
         IndexShard indexShard = getIndexShard(primaryNodeName(indexName1), indexName1);
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
index bccca283ba772..8b4981a15433a 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
@@ -56,7 +56,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase {
     protected static final String REPOSITORY_NAME = "test-remote-store-repo";
     protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2";
     protected static final int SHARD_COUNT = 1;
-    protected static final int REPLICA_COUNT = 1;
+    protected static int REPLICA_COUNT = 1;
     protected static final String TOTAL_OPERATIONS = "total-operations";
     protected static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations";
     protected static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total";
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 352d4efc95269..32396f1a3df2e 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -203,6 +203,7 @@
 import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashSet;
@@ -2006,6 +2007,29 @@ private RemoteSegmentStoreDirectory getRemoteDirectory() {
         return ((RemoteSegmentStoreDirectory) remoteDirectory);
     }
 
+    /**
+    Returns true iff it is able to verify that remote segment store
+    is in sync with local
+     */
+    boolean isRemoteSegmentStoreInSync() {
+        assert indexSettings.isRemoteStoreEnabled();
+        try {
+            RemoteSegmentStoreDirectory directory = getRemoteDirectory();
+            if (directory.readLatestMetadataFile() != null) {
+                // verifying that all files except EXCLUDE_FILES are uploaded to the remote
+                Collection uploadFiles = directory.getSegmentsUploadedToRemoteStore().keySet();
+                SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo();
+                Collection localFiles = segmentInfos.files(true);
+                if (uploadFiles.containsAll(localFiles)) {
+                    return true;
+                }
+            }
+        } catch (IOException e) {
+            logger.error("Exception while reading latest metadata", e);
+        }
+        return false;
+    }
+
     public void preRecovery() {
         final IndexShardState currentState = this.state; // single volatile read
         if (currentState == IndexShardState.CLOSED) {
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index 464adc88ae16f..dd40327298874 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -20,6 +20,7 @@
 import org.opensearch.action.LatchedActionListener;
 import org.opensearch.action.bulk.BackoffPolicy;
 import org.opensearch.action.support.GroupedActionListener;
+import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.common.concurrent.GatedCloseable;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.unit.TimeValue;
@@ -179,6 +180,9 @@ private boolean shouldSync(boolean didRefresh, boolean skipPrimaryTermCheck) {
         return this.primaryTerm != indexShard.getOperationPrimaryTerm();
     }
 
+    /*
+     @return false if retry is needed
+     */
     private boolean syncSegments() {
         if (isReadyForUpload() == false) {
             // Following check is required to enable retry and make sure that we do not lose this refresh event
@@ -485,7 +489,9 @@ private void initializeRemoteDirectoryOnTermUpdate() throws IOException {
      * @return true iff primaryMode is true and index shard is not in closed state.
      */
     private boolean isReadyForUpload() {
-        boolean isReady = indexShard.getReplicationTracker().isPrimaryMode() && indexShard.state() != IndexShardState.CLOSED;
+        boolean isReady = (indexShard.getReplicationTracker().isPrimaryMode() && indexShard.state() != IndexShardState.CLOSED)
+            || isLocalOrSnapshotRecovery();
+
         if (isReady == false) {
             StringBuilder sb = new StringBuilder("Skipped syncing segments with");
             if (indexShard.getReplicationTracker() != null) {
@@ -497,11 +503,25 @@ private boolean isReadyForUpload() {
             if (indexShard.getEngineOrNull() != null) {
                 sb.append(" engineType=").append(indexShard.getEngine().getClass().getSimpleName());
             }
+            if (isLocalOrSnapshotRecovery() == false) {
+                sb.append(" recoverySourceType=").append(indexShard.recoveryState().getRecoverySource().getType());
+                sb.append(" primary=").append(indexShard.shardRouting.primary());
+            }
             logger.trace(sb.toString());
         }
         return isReady;
     }
 
+    private boolean isLocalOrSnapshotRecovery() {
+        // In this case when the primary mode is false, we need to upload segments to Remote Store
+        // This is required in case of snapshots/shrink/ split/clone where we need to durable persist
+        // all segments to remote before completing the recovery to ensure durability.
+
+        return (indexShard.state() == IndexShardState.RECOVERING && indexShard.shardRouting.primary())
+            && (indexShard.recoveryState().getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS
+                || indexShard.recoveryState().getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT);
+    }
+
     /**
      * Creates an {@link UploadListener} containing the stats population logic which would be triggered before and after segment upload events
      */
diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
index c0211e1257c8e..e823401e5ef7e 100644
--- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
@@ -191,6 +191,15 @@ void recoverFromLocalShards(
                     // just trigger a merge to do housekeeping on the
                     // copied segments - we will also see them in stats etc.
                     indexShard.getEngine().forceMerge(false, -1, false, false, false, UUIDs.randomBase64UUID());
+                    if (indexShard.isRemoteTranslogEnabled()) {
+                        if (indexShard.isRemoteSegmentStoreInSync() == false) {
+                            throw new IndexShardRecoveryException(
+                                indexShard.shardId(),
+                                "failed to upload to remote",
+                                new IOException("Failed to upload to remote segment store")
+                            );
+                        }
+                    }
                     return true;
                 } catch (IOException ex) {
                     throw new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", ex);
@@ -418,6 +427,12 @@ void recoverFromSnapshotAndRemoteStore(
                 }
                 indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
                 indexShard.finalizeRecovery();
+                if (indexShard.isRemoteTranslogEnabled()) {
+                    if (indexShard.isRemoteSegmentStoreInSync() == false) {
+                        listener.onFailure(new IndexShardRestoreFailedException(shardId, "Failed to upload to remote segment store"));
+                        return;
+                    }
+                }
                 indexShard.postRecovery("restore done");
 
                 listener.onResponse(true);
@@ -697,6 +712,12 @@ private void restore(
             }
             indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
             indexShard.finalizeRecovery();
+            if (indexShard.isRemoteTranslogEnabled()) {
+                if (indexShard.isRemoteSegmentStoreInSync() == false) {
+                    listener.onFailure(new IndexShardRestoreFailedException(shardId, "Failed to upload to remote segment store"));
+                    return;
+                }
+            }
             indexShard.postRecovery("restore done");
             listener.onResponse(true);
         }, e -> listener.onFailure(new IndexShardRestoreFailedException(shardId, "restore failed", e)));
diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index fa3cf7676f55c..dc2111fdcfc56 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -2850,6 +2850,7 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException {
         indexDoc(source, "_doc", "1");
         indexDoc(source, "_doc", "2");
         source.refresh("test");
+        assertTrue("At lease one remote sync should have been completed", source.isRemoteSegmentStoreInSync());
         assertDocs(source, "1", "2");
         indexDoc(source, "_doc", "3");
         source.refresh("test");

From f372cbf89377036c67143360f903322105ceafe2 Mon Sep 17 00:00:00 2001
From: Aman Khare <85096200+amkhar@users.noreply.github.com>
Date: Sat, 28 Oct 2023 23:24:03 +0530
Subject: [PATCH 249/388] Add statsName field on stream while constructing
 PersistedStateStats (#10964)

Signed-off-by: Aman Khare 
Co-authored-by: Aman Khare 
---
 .../remote/RemoteClusterStateServiceIT.java   | 70 ++++++++++++++++---
 .../coordination/PersistedStateStats.java     |  8 ++-
 .../cluster/node/stats/NodeStatsTests.java    |  1 +
 3 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
index 59eef3c06844b..dcf695d5366ba 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
@@ -114,6 +114,31 @@ public void testRemoteStateStats() {
             .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
             .get();
 
+        // assert cluster state stats
+        assertClusterManagerClusterStateStats(nodesStatsResponse);
+
+        NodesStatsResponse nodesStatsResponseDataNode = client().admin()
+            .cluster()
+            .prepareNodesStats(dataNode)
+            .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
+            .get();
+        // assert cluster state stats for data node
+        DiscoveryStats dataNodeDiscoveryStats = nodesStatsResponseDataNode.getNodes().get(0).getDiscoveryStats();
+        assertNotNull(dataNodeDiscoveryStats.getClusterStateStats());
+        assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getUpdateSuccess());
+
+        // call nodes/stats with nodeId filter
+        NodesStatsResponse nodesStatsNodeIdFilterResponse = client().admin()
+            .cluster()
+            .prepareNodesStats(dataNode)
+            .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
+            .setNodesIds(clusterManagerNode)
+            .get();
+
+        assertClusterManagerClusterStateStats(nodesStatsNodeIdFilterResponse);
+    }
+
+    private void assertClusterManagerClusterStateStats(NodesStatsResponse nodesStatsResponse) {
         // assert cluster state stats
         DiscoveryStats discoveryStats = nodesStatsResponse.getNodes().get(0).getDiscoveryStats();
 
@@ -125,16 +150,43 @@ public void testRemoteStateStats() {
         assertTrue(discoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 1);
         assertEquals(0, discoveryStats.getClusterStateStats().getPersistenceStats().get(0).getFailedCount());
         assertTrue(discoveryStats.getClusterStateStats().getPersistenceStats().get(0).getTotalTimeInMillis() > 0);
+    }
 
-        NodesStatsResponse nodesStatsResponseDataNode = client().admin()
-            .cluster()
-            .prepareNodesStats(dataNode)
-            .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
-            .get();
-        // assert cluster state stats for data node
-        DiscoveryStats dataNodeDiscoveryStats = nodesStatsResponseDataNode.getNodes().get(0).getDiscoveryStats();
-        assertNotNull(dataNodeDiscoveryStats.getClusterStateStats());
-        assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getUpdateSuccess());
+    public void testRemoteStateStatsFromAllNodes() {
+        int shardCount = randomIntBetween(1, 5);
+        int replicaCount = 1;
+        int dataNodeCount = shardCount * (replicaCount + 1);
+        int clusterManagerNodeCount = 3;
+        prepareCluster(clusterManagerNodeCount, dataNodeCount, INDEX_NAME, replicaCount, shardCount);
+        String[] allNodes = internalCluster().getNodeNames();
+        // call _nodes/stats/discovery from all the nodes
+        for (String node : allNodes) {
+            NodesStatsResponse nodesStatsResponse = client().admin()
+                .cluster()
+                .prepareNodesStats(node)
+                .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
+                .get();
+            validateNodesStatsResponse(nodesStatsResponse);
+        }
+
+        // call _nodes/stats/discovery from all the nodes with random nodeId filter
+        for (String node : allNodes) {
+            NodesStatsResponse nodesStatsResponse = client().admin()
+                .cluster()
+                .prepareNodesStats(node)
+                .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName())
+                .setNodesIds(allNodes[randomIntBetween(0, allNodes.length - 1)])
+                .get();
+            validateNodesStatsResponse(nodesStatsResponse);
+        }
+    }
+
+    private void validateNodesStatsResponse(NodesStatsResponse nodesStatsResponse) {
+        // _nodes/stats/discovery must never fail due to any exception
+        assertFalse(nodesStatsResponse.toString().contains("exception"));
+        assertNotNull(nodesStatsResponse.getNodes());
+        assertNotNull(nodesStatsResponse.getNodes().get(0));
+        assertNotNull(nodesStatsResponse.getNodes().get(0).getDiscoveryStats());
     }
 
     private void setReplicaCount(int replicaCount) {
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java b/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
index 1dc20e564ade2..4d466c4b3ad73 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
@@ -25,7 +25,7 @@
  * @opensearch.internal
  */
 public class PersistedStateStats implements Writeable, ToXContentObject {
-    private String statsName;
+    private final String statsName;
     private AtomicLong totalTimeInMillis = new AtomicLong(0);
     private AtomicLong failedCount = new AtomicLong(0);
     private AtomicLong successCount = new AtomicLong(0);
@@ -37,6 +37,7 @@ public PersistedStateStats(String statsName) {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(statsName);
         out.writeVLong(successCount.get());
         out.writeVLong(failedCount.get());
         out.writeVLong(totalTimeInMillis.get());
@@ -53,6 +54,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     public PersistedStateStats(StreamInput in) throws IOException {
+        this.statsName = in.readString();
         this.successCount = new AtomicLong(in.readVLong());
         this.failedCount = new AtomicLong(in.readVLong());
         this.totalTimeInMillis = new AtomicLong(in.readVLong());
@@ -113,6 +115,10 @@ protected void addToExtendedFields(String extendedField, AtomicLong extendedFiel
         this.extendedFields.put(extendedField, extendedFieldValue);
     }
 
+    public String getStatsName() {
+        return statsName;
+    }
+
     /**
      * Fields for parsing and toXContent
      *
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 3050d1674a95b..80f4ebf5d737a 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -368,6 +368,7 @@ public void testSerialization() throws IOException {
                             .getPersistenceStats()
                             .get(0);
                         PersistedStateStats remoteStateStats = stateStats.getPersistenceStats().get(0);
+                        assertEquals(remoteStateStats.getStatsName(), deserializedRemoteStateStats.getStatsName());
                         assertEquals(remoteStateStats.getFailedCount(), deserializedRemoteStateStats.getFailedCount());
                         assertEquals(remoteStateStats.getSuccessCount(), deserializedRemoteStateStats.getSuccessCount());
                         assertEquals(remoteStateStats.getTotalTimeInMillis(), deserializedRemoteStateStats.getTotalTimeInMillis());

From 73bbeb57a8a56de0d1f9cf69e711a8a0ff26afe5 Mon Sep 17 00:00:00 2001
From: Varun Bansal 
Date: Sun, 29 Oct 2023 10:20:44 +0530
Subject: [PATCH 250/388] Restore ClusterState version during remote state
 restore (#10853)

* Restore ClusterState version during remote state restore

Signed-off-by: bansvaru 
---
 CHANGELOG.md                                  |  1 +
 .../remote/RemoteClusterStateServiceIT.java   |  4 +-
 .../RemoteStoreClusterStateRestoreIT.java     | 53 +++++++++++++++++--
 .../remote/RemoteClusterStateService.java     | 11 ++--
 .../recovery/RemoteStoreRestoreService.java   | 17 +++---
 .../RemoteClusterStateServiceTests.java       | 34 ++++++++----
 6 files changed, 93 insertions(+), 27 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 34fd573b295b3..020fb5bda8b8b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
 - [AdmissionControl] Added changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting ([#9286](https://github.com/opensearch-project/OpenSearch/pull/9286))
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
+- [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
index dcf695d5366ba..dfde1b958882c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
@@ -90,10 +90,10 @@ public void testFullClusterRestoreStaleDelete() throws Exception {
 
         assertEquals(10, repository.blobStore().blobContainer(baseMetadataPath.add("manifest")).listBlobsByPrefix("manifest").size());
 
-        Map indexMetadataMap = remoteClusterStateService.getLatestMetadata(
+        Map indexMetadataMap = remoteClusterStateService.getLatestClusterState(
             cluster().getClusterName(),
             getClusterState().metadata().clusterUUID()
-        ).getIndices();
+        ).getMetadata().getIndices();
         assertEquals(0, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas());
         assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards());
     }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
index e9afd6d36bb87..c61e2ec6e4f6c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -30,6 +30,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
@@ -85,6 +86,7 @@ public void testFullClusterRestore() throws Exception {
         // Step - 1 index some data to generate files in remote directory
         Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, 1);
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+        long prevClusterStateVersion = clusterService().state().version();
 
         // Step - 2 Replace all nodes in the cluster with new nodes. This ensures new cluster state doesn't have previous index metadata
         resetCluster(dataNodeCount, clusterManagerNodeCount);
@@ -92,9 +94,17 @@ public void testFullClusterRestore() throws Exception {
         String newClusterUUID = clusterService().state().metadata().clusterUUID();
         assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
 
-        // Step - 3 Trigger full cluster restore and validate
+        // Step - 3 validate cluster state restored
+        long newClusterStateVersion = clusterService().state().version();
+        assert prevClusterStateVersion < newClusterStateVersion : String.format(
+            Locale.ROOT,
+            "ClusterState version is not restored. previousClusterVersion: [%s] is greater than current [%s]",
+            prevClusterStateVersion,
+            newClusterStateVersion
+        );
         validateMetadata(List.of(INDEX_NAME));
         verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true);
+
     }
 
     /**
@@ -121,6 +131,7 @@ public void testFullClusterRestoreDoesntFailWithConflictingLocalState() throws E
         // index some data to generate files in remote directory
         Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, 1);
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+        long prevClusterStateVersion = clusterService().state().version();
 
         // stop all nodes
         internalCluster().stopAllNodes();
@@ -156,6 +167,14 @@ public Settings onNodeStopped(String nodeName) {
         newClusterUUID = clusterService().state().metadata().clusterUUID();
         assert !Objects.equals(newClusterUUID, ClusterState.UNKNOWN_UUID) : "cluster restart not successful. cluster uuid is still unknown";
         assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
+
+        long newClusterStateVersion = clusterService().state().version();
+        assert prevClusterStateVersion < newClusterStateVersion : String.format(
+            Locale.ROOT,
+            "ClusterState version is not restored. previousClusterVersion: [%s] is greater than current [%s]",
+            prevClusterStateVersion,
+            newClusterStateVersion
+        );
         validateMetadata(List.of(INDEX_NAME));
 
         // start data nodes to trigger index data recovery
@@ -180,6 +199,7 @@ public void testFullClusterRestoreMultipleIndices() throws Exception {
         updateIndexBlock(true, secondIndexName);
 
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+        long prevClusterStateVersion = clusterService().state().version();
 
         // Step - 2 Replace all nodes in the cluster with new nodes. This ensures new cluster state doesn't have previous index metadata
         resetCluster(dataNodeCount, clusterManagerNodeCount);
@@ -187,7 +207,14 @@ public void testFullClusterRestoreMultipleIndices() throws Exception {
         String newClusterUUID = clusterService().state().metadata().clusterUUID();
         assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
 
-        // Step - 3 Trigger full cluster restore
+        // Step - 3 validate cluster state restored
+        long newClusterStateVersion = clusterService().state().version();
+        assert prevClusterStateVersion < newClusterStateVersion : String.format(
+            Locale.ROOT,
+            "ClusterState version is not restored. previousClusterVersion: [%s] is greater than current [%s]",
+            prevClusterStateVersion,
+            newClusterStateVersion
+        );
         validateMetadata(List.of(INDEX_NAME, secondIndexName));
         verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, false);
         verifyRedIndicesAndTriggerRestore(indexStats2, secondIndexName, false);
@@ -239,6 +266,7 @@ public void testRemoteStateFullRestart() throws Exception {
 
         Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount);
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+        long prevClusterStateVersion = clusterService().state().version();
         // Delete index metadata file in remote
         try {
             Files.move(
@@ -257,6 +285,14 @@ public void testRemoteStateFullRestart() throws Exception {
         ensureGreen(INDEX_NAME);
         String newClusterUUID = clusterService().state().metadata().clusterUUID();
         assert Objects.equals(newClusterUUID, prevClusterUUID) : "Full restart not successful. cluster uuid has changed";
+
+        long newClusterStateVersion = clusterService().state().version();
+        assert prevClusterStateVersion < newClusterStateVersion : String.format(
+            Locale.ROOT,
+            "ClusterState version is not restored. previousClusterVersion: [%s] is greater than current [%s]",
+            prevClusterStateVersion,
+            newClusterStateVersion
+        );
         validateCurrentMetadata();
         verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true);
     }
@@ -309,6 +345,7 @@ public void testFullClusterRestoreGlobalMetadata() throws Exception {
         // Step - 1 index some data to generate files in remote directory
         Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, 1);
         String prevClusterUUID = clusterService().state().metadata().clusterUUID();
+        long prevClusterStateVersion = clusterService().state().version();
 
         // Create global metadata - register a custom repo
         Path repoPath = registerCustomRepository();
@@ -328,8 +365,16 @@ public void testFullClusterRestoreGlobalMetadata() throws Exception {
         String newClusterUUID = clusterService().state().metadata().clusterUUID();
         assert !Objects.equals(newClusterUUID, prevClusterUUID) : "cluster restart not successful. cluster uuid is same";
 
-        // Step - 3 Trigger full cluster restore and validate
-        // validateCurrentMetadata();
+        // Step - 3 validate cluster state restored
+        long newClusterStateVersion = clusterService().state().version();
+        assert prevClusterStateVersion < newClusterStateVersion : String.format(
+            Locale.ROOT,
+            "ClusterState version is not restored. previousClusterVersion: [%s] is greater than current [%s]",
+            prevClusterStateVersion,
+            newClusterStateVersion
+        );
+
+        validateCurrentMetadata();
         assertEquals(Integer.valueOf(34), SETTING_CLUSTER_MAX_SHARDS_PER_NODE.get(clusterService().state().metadata().settings()));
         assertEquals(true, SETTING_READ_ONLY_SETTING.get(clusterService().state().metadata().settings()));
         assertTrue(clusterService().state().blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK));
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index b3309b1fd8a63..205ae12cf6214 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -767,16 +767,16 @@ private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, U
     }
 
     /**
-     * Fetch latest metadata from remote cluster state including global metadata and index metadata
+     * Fetch latest ClusterState from remote, including global metadata, index metadata and cluster state version
      *
      * @param clusterUUID uuid of cluster state to refer to in remote
      * @param clusterName name of the cluster
      * @return {@link IndexMetadata}
      */
-    public Metadata getLatestMetadata(String clusterName, String clusterUUID) {
+    public ClusterState getLatestClusterState(String clusterName, String clusterUUID) {
         start();
         Optional clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
-        if (!clusterMetadataManifest.isPresent()) {
+        if (clusterMetadataManifest.isEmpty()) {
             throw new IllegalStateException(
                 String.format(Locale.ROOT, "Latest cluster metadata manifest is not present for the provided clusterUUID: %s", clusterUUID)
             );
@@ -790,7 +790,10 @@ public Metadata getLatestMetadata(String clusterName, String clusterUUID) {
         Map indexMetadataMap = new HashMap<>();
         indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); });
 
-        return Metadata.builder(globalMetadata).indices(indexMetadataMap).build();
+        return ClusterState.builder(ClusterState.EMPTY_STATE)
+            .version(clusterMetadataManifest.get().getStateVersion())
+            .metadata(Metadata.builder(globalMetadata).indices(indexMetadataMap).build())
+            .build();
     }
 
     private Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) {
diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
index aebd7d2ea201a..23bb4cea17a20 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
@@ -138,7 +138,7 @@ public RemoteRestoreResult restore(
         String[] indexNames
     ) {
         Map> indexMetadataMap = new HashMap<>();
-        Metadata remoteMetadata = null;
+        ClusterState remoteState = null;
         boolean metadataFromRemoteStore = (restoreClusterUUID == null
             || restoreClusterUUID.isEmpty()
             || restoreClusterUUID.isBlank()) == false;
@@ -150,8 +150,8 @@ public RemoteRestoreResult restore(
                     throw new IllegalArgumentException("clusterUUID to restore from should be different from current cluster UUID");
                 }
                 logger.info("Restoring cluster state from remote store from cluster UUID : [{}]", restoreClusterUUID);
-                remoteMetadata = remoteClusterStateService.getLatestMetadata(currentState.getClusterName().value(), restoreClusterUUID);
-                remoteMetadata.getIndices().values().forEach(indexMetadata -> {
+                remoteState = remoteClusterStateService.getLatestClusterState(currentState.getClusterName().value(), restoreClusterUUID);
+                remoteState.getMetadata().getIndices().values().forEach(indexMetadata -> {
                     indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));
                 });
             } catch (Exception e) {
@@ -177,7 +177,7 @@ public RemoteRestoreResult restore(
                 }
             }
         }
-        return executeRestore(currentState, indexMetadataMap, restoreAllShards, remoteMetadata);
+        return executeRestore(currentState, indexMetadataMap, restoreAllShards, remoteState);
     }
 
     /**
@@ -191,7 +191,7 @@ private RemoteRestoreResult executeRestore(
         ClusterState currentState,
         Map> indexMetadataMap,
         boolean restoreAllShards,
-        Metadata remoteMetadata
+        ClusterState remoteState
     ) {
         final String restoreUUID = UUIDs.randomBase64UUID();
         List indicesToBeRestored = new ArrayList<>();
@@ -241,8 +241,11 @@ private RemoteRestoreResult executeRestore(
             totalShards += updatedIndexMetadata.getNumberOfShards();
         }
 
-        if (remoteMetadata != null) {
-            restoreGlobalMetadata(mdBuilder, remoteMetadata);
+        if (remoteState != null) {
+            restoreGlobalMetadata(mdBuilder, remoteState.getMetadata());
+            // Restore ClusterState version
+            logger.info("Restoring ClusterState with Remote State version [{}]", remoteState.version());
+            builder.version(remoteState.version());
         }
 
         RestoreInfo restoreInfo = new RestoreInfo("remote_store", indicesToBeRestored, totalShards, totalShards);
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 586618bd1ecff..4efd1b8a62970 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -665,7 +665,8 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE
 
         remoteClusterStateService.start();
         assertEquals(
-            remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+            remoteClusterStateService.getLatestClusterState(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+                .getMetadata()
                 .getIndices()
                 .size(),
             0
@@ -694,8 +695,10 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
         remoteClusterStateService.start();
         Exception e = assertThrows(
             IllegalStateException.class,
-            () -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
-                .getIndices()
+            () -> remoteClusterStateService.getLatestClusterState(
+                clusterState.getClusterName().value(),
+                clusterState.metadata().clusterUUID()
+            ).getMetadata().getIndices()
         );
         assertEquals(e.getMessage(), "Error while downloading IndexMetadata - " + uploadedIndexMetadata.getUploadedFilename());
     }
@@ -740,10 +743,11 @@ public void testReadGlobalMetadata() throws IOException {
         final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build();
         remoteClusterStateService.start();
 
+        long prevClusterStateVersion = 13L;
         final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder()
             .indices(List.of())
             .clusterTerm(1L)
-            .stateVersion(1L)
+            .stateVersion(prevClusterStateVersion)
             .stateUUID("state-uuid")
             .clusterUUID("cluster-uuid")
             .codecVersion(MANIFEST_CURRENT_CODEC_VERSION)
@@ -756,12 +760,20 @@ public void testReadGlobalMetadata() throws IOException {
         Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build();
         mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expactedMetadata);
 
-        Metadata metadata = remoteClusterStateService.getLatestMetadata(
+        ClusterState newClusterState = remoteClusterStateService.getLatestClusterState(
             clusterState.getClusterName().value(),
             clusterState.metadata().clusterUUID()
         );
 
-        assertTrue(Metadata.isGlobalStateEquals(metadata, expactedMetadata));
+        assertTrue(Metadata.isGlobalStateEquals(newClusterState.getMetadata(), expactedMetadata));
+
+        long newClusterStateVersion = newClusterState.getVersion();
+        assert prevClusterStateVersion == newClusterStateVersion : String.format(
+            Locale.ROOT,
+            "ClusterState version is not restored. previousClusterVersion: [%s] is not equal to current [%s]",
+            prevClusterStateVersion,
+            newClusterStateVersion
+        );
     }
 
     public void testReadGlobalMetadataIOException() throws IOException {
@@ -793,7 +805,10 @@ public void testReadGlobalMetadataIOException() throws IOException {
         remoteClusterStateService.start();
         Exception e = assertThrows(
             IllegalStateException.class,
-            () -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+            () -> remoteClusterStateService.getLatestClusterState(
+                clusterState.getClusterName().value(),
+                clusterState.metadata().clusterUUID()
+            )
         );
         assertEquals(e.getMessage(), "Error while downloading Global Metadata - " + globalIndexMetadataName);
     }
@@ -824,16 +839,15 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
-            .globalMetadataFileName("global-metadata-file")
             .codecVersion(ClusterMetadataManifest.CODEC_V0)
             .build();
 
         mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
 
-        Map indexMetadataMap = remoteClusterStateService.getLatestMetadata(
+        Map indexMetadataMap = remoteClusterStateService.getLatestClusterState(
             clusterState.getClusterName().value(),
             clusterState.metadata().clusterUUID()
-        ).getIndices();
+        ).getMetadata().getIndices();
 
         assertEquals(indexMetadataMap.size(), 1);
         assertEquals(indexMetadataMap.get(index.getName()).getIndex().getName(), index.getName());

From 84be8c9207cf1153b2eb8dfaf77cf737959781cc Mon Sep 17 00:00:00 2001
From: Dhwanil Patel 
Date: Mon, 30 Oct 2023 12:55:01 +0530
Subject: [PATCH 251/388] Use async write for manifest file and use latch for
 timeout (#10968)

* Use async write for manifest file and use latch for timeout

Signed-off-by: Dhwanil Patel 
---
 .../common/settings/ClusterSettings.java      |  1 +
 .../remote/RemoteClusterStateService.java     | 91 +++++++++++++------
 .../RemoteClusterStateServiceTests.java       | 84 ++++++++++++++---
 3 files changed, 137 insertions(+), 39 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index c2c6effc3336f..3a1fff21db366 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -684,6 +684,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING,
                 RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING,
                 RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING,
+                RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING,
                 RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
                 IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
                 IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index 205ae12cf6214..c892b475d71da 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -87,6 +87,8 @@ public class RemoteClusterStateService implements Closeable {
 
     public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000);
 
+    public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000);
+
     public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting(
         "cluster.remote_store.state.index_metadata.upload_timeout",
         INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT,
@@ -101,6 +103,13 @@ public class RemoteClusterStateService implements Closeable {
         Setting.Property.NodeScope
     );
 
+    public static final Setting METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting(
+        "cluster.remote_store.state.metadata_manifest.upload_timeout",
+        METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
     public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
         "index-metadata",
         METADATA_NAME_FORMAT,
@@ -157,6 +166,7 @@ public class RemoteClusterStateService implements Closeable {
 
     private volatile TimeValue indexMetadataUploadTimeout;
     private volatile TimeValue globalMetadataUploadTimeout;
+    private volatile TimeValue metadataManifestUploadTimeout;
 
     private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false);
     private final RemotePersistenceStats remoteStateStats;
@@ -190,9 +200,11 @@ public RemoteClusterStateService(
         this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD);
         this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING);
         this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING);
+        this.metadataManifestUploadTimeout = clusterSettings.get(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING);
         clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold);
         clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout);
         clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout);
+        clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout);
         this.remoteStateStats = new RemotePersistenceStats();
     }
 
@@ -401,13 +413,13 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException
         try {
             if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
                 // TODO: We should add metrics where transfer is timing out. [Issue: #10687]
-                GlobalMetadataTransferException ex = new GlobalMetadataTransferException(
+                RemoteStateTransferException ex = new RemoteStateTransferException(
                     String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete")
                 );
                 throw ex;
             }
         } catch (InterruptedException ex) {
-            GlobalMetadataTransferException exception = new GlobalMetadataTransferException(
+            RemoteStateTransferException exception = new RemoteStateTransferException(
                 String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete - %s"),
                 ex
             );
@@ -415,7 +427,7 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException
             throw exception;
         }
         if (exceptionReference.get() != null) {
-            throw new GlobalMetadataTransferException(exceptionReference.get().getMessage(), exceptionReference.get());
+            throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get());
         }
         return result.get();
     }
@@ -440,7 +452,7 @@ private List writeIndexMetadataParallel(ClusterState clus
                 );
                 result.add(uploadedIndexMetadata);
             }, ex -> {
-                assert ex instanceof IndexMetadataTransferException;
+                assert ex instanceof RemoteStateTransferException;
                 logger.error(
                     () -> new ParameterizedMessage("Exception during transfer of IndexMetadata to Remote {}", ex.getMessage()),
                     ex
@@ -457,7 +469,7 @@ private List writeIndexMetadataParallel(ClusterState clus
 
         try {
             if (latch.await(getIndexMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
-                IndexMetadataTransferException ex = new IndexMetadataTransferException(
+                RemoteStateTransferException ex = new RemoteStateTransferException(
                     String.format(
                         Locale.ROOT,
                         "Timed out waiting for transfer of index metadata to complete - %s",
@@ -469,7 +481,7 @@ private List writeIndexMetadataParallel(ClusterState clus
             }
         } catch (InterruptedException ex) {
             exceptionList.forEach(ex::addSuppressed);
-            IndexMetadataTransferException exception = new IndexMetadataTransferException(
+            RemoteStateTransferException exception = new RemoteStateTransferException(
                 String.format(
                     Locale.ROOT,
                     "Timed out waiting for transfer of index metadata to complete - %s",
@@ -481,7 +493,7 @@ private List writeIndexMetadataParallel(ClusterState clus
             throw exception;
         }
         if (exceptionList.size() > 0) {
-            IndexMetadataTransferException exception = new IndexMetadataTransferException(
+            RemoteStateTransferException exception = new RemoteStateTransferException(
                 String.format(
                     Locale.ROOT,
                     "Exception during transfer of IndexMetadata to Remote %s",
@@ -520,7 +532,7 @@ private void writeIndexMetadataAsync(
                     indexMetadataContainer.path().buildAsString() + indexMetadataFilename
                 )
             ),
-            ex -> latchedActionListener.onFailure(new IndexMetadataTransferException(indexMetadata.getIndex().toString(), ex))
+            ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex))
         );
 
         INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority(
@@ -601,14 +613,45 @@ private ClusterMetadataManifest uploadManifest(
 
     private void writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest, String fileName)
         throws IOException {
+        AtomicReference result = new AtomicReference();
+        AtomicReference exceptionReference = new AtomicReference();
+
         final BlobContainer metadataManifestContainer = manifestContainer(clusterName, clusterUUID);
-        CLUSTER_METADATA_MANIFEST_FORMAT.write(
+
+        // latch to wait until upload is not finished
+        CountDownLatch latch = new CountDownLatch(1);
+
+        LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> {
+            logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully."));
+        }, ex -> { exceptionReference.set(ex); }), latch);
+
+        CLUSTER_METADATA_MANIFEST_FORMAT.writeAsyncWithUrgentPriority(
             uploadManifest,
             metadataManifestContainer,
             fileName,
             blobStoreRepository.getCompressor(),
+            completionListener,
             FORMAT_PARAMS
         );
+
+        try {
+            if (latch.await(getMetadataManifestUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
+                RemoteStateTransferException ex = new RemoteStateTransferException(
+                    String.format(Locale.ROOT, "Timed out waiting for transfer of manifest file to complete")
+                );
+                throw ex;
+            }
+        } catch (InterruptedException ex) {
+            RemoteStateTransferException exception = new RemoteStateTransferException(
+                String.format(Locale.ROOT, "Timed out waiting for transfer of manifest file to complete - %s"),
+                ex
+            );
+            Thread.currentThread().interrupt();
+            throw exception;
+        }
+        if (exceptionReference.get() != null) {
+            throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get());
+        }
         logger.debug(
             "Metadata manifest file [{}] written during [{}] phase. ",
             fileName,
@@ -668,6 +711,10 @@ private void setGlobalMetadataUploadTimeout(TimeValue newGlobalMetadataUploadTim
         this.globalMetadataUploadTimeout = newGlobalMetadataUploadTimeout;
     }
 
+    private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploadTimeout) {
+        this.metadataManifestUploadTimeout = newMetadataManifestUploadTimeout;
+    }
+
     public TimeValue getIndexMetadataUploadTimeout() {
         return this.indexMetadataUploadTimeout;
     }
@@ -676,6 +723,10 @@ public TimeValue getGlobalMetadataUploadTimeout() {
         return this.globalMetadataUploadTimeout;
     }
 
+    public TimeValue getMetadataManifestUploadTimeout() {
+        return this.metadataManifestUploadTimeout;
+    }
+
     static String getManifestFileName(long term, long version, boolean committed) {
         // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____
         return String.join(
@@ -1088,29 +1139,15 @@ public void writeMetadataFailed() {
     }
 
     /**
-     * Exception for IndexMetadata transfer failures to remote
-     */
-    static class IndexMetadataTransferException extends RuntimeException {
-
-        public IndexMetadataTransferException(String errorDesc) {
-            super(errorDesc);
-        }
-
-        public IndexMetadataTransferException(String errorDesc, Throwable cause) {
-            super(errorDesc, cause);
-        }
-    }
-
-    /**
-     * Exception for GlobalMetadata transfer failures to remote
+     * Exception for Remote state transfer.
      */
-    static class GlobalMetadataTransferException extends RuntimeException {
+    static class RemoteStateTransferException extends RuntimeException {
 
-        public GlobalMetadataTransferException(String errorDesc) {
+        public RemoteStateTransferException(String errorDesc) {
             super(errorDesc);
         }
 
-        public GlobalMetadataTransferException(String errorDesc, Throwable cause) {
+        public RemoteStateTransferException(String errorDesc, Throwable cause) {
             super(errorDesc, cause);
         }
     }
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 4efd1b8a62970..65477051cdb30 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -68,6 +68,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
@@ -230,10 +231,17 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
 
         ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
         ArgumentCaptor writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class);
-
+        AtomicReference capturedWriteContext = new AtomicReference<>();
         doAnswer((i) -> {
             actionListenerArgumentCaptor.getValue().onResponse(null);
             return null;
+        }).doAnswer((i) -> {
+            actionListenerArgumentCaptor.getValue().onResponse(null);
+            capturedWriteContext.set(writeContextArgumentCaptor.getValue());
+            return null;
+        }).doAnswer((i) -> {
+            actionListenerArgumentCaptor.getValue().onResponse(null);
+            return null;
         }).when(container).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture());
 
         remoteClusterStateService.start();
@@ -262,27 +270,30 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException {
         assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID()));
         assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID()));
 
-        assertEquals(actionListenerArgumentCaptor.getAllValues().size(), 2);
-        assertEquals(writeContextArgumentCaptor.getAllValues().size(), 2);
+        assertEquals(actionListenerArgumentCaptor.getAllValues().size(), 3);
+        assertEquals(writeContextArgumentCaptor.getAllValues().size(), 3);
 
-        WriteContext capturedWriteContext = writeContextArgumentCaptor.getValue();
-        byte[] writtenBytes = capturedWriteContext.getStreamProvider(Integer.MAX_VALUE).provideStream(0).getInputStream().readAllBytes();
+        byte[] writtenBytes = capturedWriteContext.get()
+            .getStreamProvider(Integer.MAX_VALUE)
+            .provideStream(0)
+            .getInputStream()
+            .readAllBytes();
         IndexMetadata writtenIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.deserialize(
-            capturedWriteContext.getFileName(),
+            capturedWriteContext.get().getFileName(),
             blobStoreRepository.getNamedXContentRegistry(),
             new BytesArray(writtenBytes)
         );
 
-        assertEquals(capturedWriteContext.getWritePriority(), WritePriority.URGENT);
+        assertEquals(capturedWriteContext.get().getWritePriority(), WritePriority.URGENT);
         assertEquals(writtenIndexMetadata.getNumberOfShards(), 1);
         assertEquals(writtenIndexMetadata.getNumberOfReplicas(), 0);
         assertEquals(writtenIndexMetadata.getIndex().getName(), "test-index");
         assertEquals(writtenIndexMetadata.getIndex().getUUID(), "index-uuid");
         long expectedChecksum = RemoteTransferContainer.checksumOfChecksum(new ByteArrayIndexInput("metadata-filename", writtenBytes), 8);
-        if (capturedWriteContext.doRemoteDataIntegrityCheck()) {
-            assertEquals(capturedWriteContext.getExpectedChecksum().longValue(), expectedChecksum);
+        if (capturedWriteContext.get().doRemoteDataIntegrityCheck()) {
+            assertEquals(capturedWriteContext.get().getExpectedChecksum().longValue(), expectedChecksum);
         } else {
-            assertEquals(capturedWriteContext.getExpectedChecksum(), null);
+            assertEquals(capturedWriteContext.get().getExpectedChecksum(), null);
         }
 
     }
@@ -306,11 +317,44 @@ public void run() {
 
         remoteClusterStateService.start();
         assertThrows(
-            RemoteClusterStateService.GlobalMetadataTransferException.class,
+            RemoteClusterStateService.RemoteStateTransferException.class,
             () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10))
         );
     }
 
+    public void testTimeoutWhileWritingManifestFile() throws IOException {
+        // verify update metadata manifest upload timeout
+        int metadataManifestUploadTimeout = 2;
+        Settings newSettings = Settings.builder()
+            .put("cluster.remote_store.state.metadata_manifest.upload_timeout", metadataManifestUploadTimeout + "s")
+            .build();
+        clusterSettings.applySettings(newSettings);
+
+        final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
+        AsyncMultiStreamBlobContainer container = (AsyncMultiStreamBlobContainer) mockBlobStoreObjects(AsyncMultiStreamBlobContainer.class);
+
+        ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class);
+
+        doAnswer((i) -> { // For Global Metadata
+            actionListenerArgumentCaptor.getValue().onResponse(null);
+            return null;
+        }).doAnswer((i) -> { // For Index Metadata
+            actionListenerArgumentCaptor.getValue().onResponse(null);
+            return null;
+        }).doAnswer((i) -> {
+            // For Manifest file perform No Op, so latch in code will timeout
+            return null;
+        }).when(container).asyncBlobUpload(any(WriteContext.class), actionListenerArgumentCaptor.capture());
+
+        remoteClusterStateService.start();
+        try {
+            remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10));
+        } catch (Exception e) {
+            assertTrue(e instanceof RemoteClusterStateService.RemoteStateTransferException);
+            assertTrue(e.getMessage().contains("Timed out waiting for transfer of manifest file to complete"));
+        }
+    }
+
     public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOException {
         final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build();
         AsyncMultiStreamBlobContainer container = (AsyncMultiStreamBlobContainer) mockBlobStoreObjects(AsyncMultiStreamBlobContainer.class);
@@ -327,7 +371,7 @@ public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOEx
 
         remoteClusterStateService.start();
         assertThrows(
-            RemoteClusterStateService.IndexMetadataTransferException.class,
+            RemoteClusterStateService.RemoteStateTransferException.class,
             () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10))
         );
         assertEquals(0, remoteClusterStateService.getStats().getSuccessCount());
@@ -1142,6 +1186,22 @@ public void testIndexMetadataUploadWaitTimeSetting() {
         assertEquals(indexMetadataUploadTimeout, remoteClusterStateService.getIndexMetadataUploadTimeout().seconds());
     }
 
+    public void testMetadataManifestUploadWaitTimeSetting() {
+        // verify default value
+        assertEquals(
+            RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT,
+            remoteClusterStateService.getMetadataManifestUploadTimeout()
+        );
+
+        // verify update metadata manifest upload timeout
+        int metadataManifestUploadTimeout = randomIntBetween(1, 10);
+        Settings newSettings = Settings.builder()
+            .put("cluster.remote_store.state.metadata_manifest.upload_timeout", metadataManifestUploadTimeout + "s")
+            .build();
+        clusterSettings.applySettings(newSettings);
+        assertEquals(metadataManifestUploadTimeout, remoteClusterStateService.getMetadataManifestUploadTimeout().seconds());
+    }
+
     public void testGlobalMetadataUploadWaitTimeSetting() {
         // verify default value
         assertEquals(

From 4efa6d7a8dfdaba93571785315eeeb24956917f5 Mon Sep 17 00:00:00 2001
From: Sachin Kale 
Date: Mon, 30 Oct 2023 14:58:59 +0530
Subject: [PATCH 252/388] Read the same medata file that is locked during
 restore of shallow snapshot (#10979)

Signed-off-by: Sachin Kale 
---
 .../remotestore/RemoteRestoreSnapshotIT.java  | 67 +++++++++++++++++++
 .../opensearch/index/shard/IndexShard.java    |  3 +-
 .../opensearch/index/shard/StoreRecovery.java |  5 ++
 .../store/RemoteSegmentStoreDirectory.java    |  6 +-
 .../RemoteStoreMetadataLockManager.java       | 16 +++++
 5 files changed, 93 insertions(+), 4 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
index ad78c503a4a19..21ce4be9981fb 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
@@ -589,4 +589,71 @@ public void testRestoreShallowSnapshotRepository() throws ExecutionException, In
         assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
     }
 
+    public void testRestoreShallowSnapshotIndexAfterSnapshot() throws ExecutionException, InterruptedException {
+        String indexName1 = "testindex1";
+        String snapshotRepoName = "test-restore-snapshot-repo";
+        String remoteStoreRepoNameUpdated = "test-rs-repo-updated" + TEST_REMOTE_STORE_REPO_SUFFIX;
+        String snapshotName1 = "test-restore-snapshot1";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        Path absolutePath2 = randomRepoPath().toAbsolutePath();
+        String[] pathTokens = absolutePath1.toString().split("/");
+        String basePath = pathTokens[pathTokens.length - 1];
+        Arrays.copyOf(pathTokens, pathTokens.length - 1);
+        Path location = PathUtils.get(String.join("/", pathTokens));
+        pathTokens = absolutePath2.toString().split("/");
+        String basePath2 = pathTokens[pathTokens.length - 1];
+        Arrays.copyOf(pathTokens, pathTokens.length - 1);
+        Path location2 = PathUtils.get(String.join("/", pathTokens));
+        logger.info("Path 1 [{}]", absolutePath1);
+        logger.info("Path 2 [{}]", absolutePath2);
+        String restoredIndexName1 = indexName1 + "-restored";
+
+        createRepository(snapshotRepoName, "fs", getRepositorySettings(location, basePath, true));
+
+        Client client = client();
+        Settings indexSettings = Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+        createIndex(indexName1, indexSettings);
+
+        int numDocsInIndex1 = randomIntBetween(2, 5);
+        indexDocuments(client, indexName1, numDocsInIndex1);
+
+        ensureGreen(indexName1);
+
+        logger.info("--> snapshot");
+        SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(List.of(indexName1)));
+        assertThat(snapshotInfo1.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards()));
+        assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS));
+
+        int extraNumDocsInIndex1 = randomIntBetween(20, 50);
+        indexDocuments(client, indexName1, extraNumDocsInIndex1);
+        refresh(indexName1);
+
+        client().admin().indices().close(Requests.closeIndexRequest(indexName1)).get();
+        createRepository(remoteStoreRepoNameUpdated, "fs", remoteRepoPath);
+        RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
+            .setWaitForCompletion(true)
+            .setIndices(indexName1)
+            .setRenamePattern(indexName1)
+            .setRenameReplacement(restoredIndexName1)
+            .setSourceRemoteStoreRepository(remoteStoreRepoNameUpdated)
+            .get();
+
+        assertTrue(restoreSnapshotResponse2.getRestoreInfo().failedShards() == 0);
+        ensureGreen(restoredIndexName1);
+        assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1);
+
+        // indexing some new docs and validating
+        indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2);
+        ensureGreen(restoredIndexName1);
+        assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 32396f1a3df2e..cf42c6749fc79 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -4908,8 +4908,7 @@ public void syncSegmentsFromGivenRemoteSegmentStore(
             remoteStore.incRef();
         }
         Map uploadedSegments = sourceRemoteDirectory
-            .initializeToSpecificCommit(primaryTerm, commitGeneration)
-            .getMetadata();
+            .getSegmentsUploadedToRemoteStore();
         final Directory storeDirectory = store.directory();
         store.incRef();
 
diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
index e823401e5ef7e..5b1940bb1d9a5 100644
--- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
@@ -410,6 +410,11 @@ void recoverFromSnapshotAndRemoteStore(
                     indexUUID,
                     shardId
                 );
+                sourceRemoteDirectory.initializeToSpecificCommit(
+                    primaryTerm,
+                    commitGeneration,
+                    recoverySource.snapshot().getSnapshotId().getUUID()
+                );
                 indexShard.syncSegmentsFromGivenRemoteSegmentStore(true, sourceRemoteDirectory, primaryTerm, commitGeneration);
                 final Store store = indexShard.store();
                 if (indexShard.indexSettings.isRemoteTranslogStoreEnabled() == false) {
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
index be1f2341236ab..988d52202f975 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
@@ -34,6 +34,7 @@
 import org.opensearch.index.store.lockmanager.FileLockInfo;
 import org.opensearch.index.store.lockmanager.RemoteStoreCommitLevelLockManager;
 import org.opensearch.index.store.lockmanager.RemoteStoreLockManager;
+import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager;
 import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata;
 import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
@@ -160,8 +161,9 @@ public RemoteSegmentMetadata init() throws IOException {
      *
      * @throws IOException if there were any failures in reading the metadata file
      */
-    public RemoteSegmentMetadata initializeToSpecificCommit(long primaryTerm, long commitGeneration) throws IOException {
-        String metadataFile = getMetadataFileForCommit(primaryTerm, commitGeneration);
+    public RemoteSegmentMetadata initializeToSpecificCommit(long primaryTerm, long commitGeneration, String acquirerId) throws IOException {
+        String metadataFilePrefix = MetadataFilenameUtils.getMetadataFilePrefixForCommit(primaryTerm, commitGeneration);
+        String metadataFile = ((RemoteStoreMetadataLockManager) mdLockManager).fetchLock(metadataFilePrefix, acquirerId);
         RemoteSegmentMetadata remoteSegmentMetadata = readMetadataFile(metadataFile);
         if (remoteSegmentMetadata != null) {
             this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(remoteSegmentMetadata.getMetadata());
diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java
index fd7906729e314..756905d02229a 100644
--- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java
+++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java
@@ -14,10 +14,13 @@
 import org.apache.lucene.store.IndexOutput;
 import org.opensearch.index.store.RemoteBufferedOutputDirectory;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.NoSuchFileException;
 import java.util.Collection;
+import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * A Class that implements Remote Store Lock Manager by creating lock files for the remote store files that needs to
@@ -70,6 +73,19 @@ public void release(LockInfo lockInfo) throws IOException {
         }
     }
 
+    public String fetchLock(String filenamePrefix, String acquirerId) throws IOException {
+        Collection lockFiles = lockDirectory.listFilesByPrefix(filenamePrefix);
+        List lockFilesForAcquirer = lockFiles.stream()
+            .filter(lockFile -> acquirerId.equals(FileLockInfo.LockFileUtils.getAcquirerIdFromLock(lockFile)))
+            .map(FileLockInfo.LockFileUtils::getFileToLockNameFromLock)
+            .collect(Collectors.toList());
+        if (lockFilesForAcquirer.size() == 0) {
+            throw new FileNotFoundException("No lock file found for prefix: " + filenamePrefix + " and acquirerId: " + acquirerId);
+        }
+        assert lockFilesForAcquirer.size() == 1;
+        return lockFilesForAcquirer.get(0);
+    }
+
     /**
      * Checks whether a given file have any lock on it or not.
      * @param lockInfo File Lock Info instance for which we need to check if lock is acquired.

From 0d7d1e9db7ace8f6e90ab98ade88d719efaf37e1 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Mon, 30 Oct 2023 09:15:17 -0400
Subject: [PATCH 253/388] Update bundled JDK to JDK-21.0.1 (#10576)

Signed-off-by: Andriy Redko 
---
 .../java/org/opensearch/gradle/test/DistroTestPlugin.java     | 4 ++--
 buildSrc/version.properties                                   | 4 +---
 distribution/src/config/jvm.options                           | 2 +-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
index a420c8b63b02c..1ad7e056b6ae6 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin {
-    private static final String SYSTEM_JDK_VERSION = "11.0.20+8";
+    private static final String SYSTEM_JDK_VERSION = "17.0.9+9";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.8+7";
+    private static final String GRADLE_JDK_VERSION = "17.0.9+9";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 96d398c35851d..0d98cba35448f 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -2,9 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.8.0
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 20.0.2+9
-# See please https://github.com/adoptium/temurin-build/issues/3371
-bundled_jdk_linux_ppc64le = 20+36
+bundled_jdk = 21.0.1+12
 
 # optional dependencies
 spatial4j         = 0.7
diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index 952110c6c0289..1a0abcbaf9c88 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -81,7 +81,7 @@ ${error.file}
 
 # JDK 20+ Incubating Vector Module for SIMD optimizations;
 # disabling may reduce performance on vector optimized lucene
-20:--add-modules=jdk.incubator.vector
+20-:--add-modules=jdk.incubator.vector
 
 # HDFS ForkJoinPool.common() support by SecurityManager
 -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory

From 7c917c5628d8c33a7316bd30e7b8c5c70419f8a1 Mon Sep 17 00:00:00 2001
From: Sachin Kale 
Date: Mon, 30 Oct 2023 19:06:51 +0530
Subject: [PATCH 254/388] Log more info with max seq no on upload to and
 download from remote translog (#10973)

* Log more info with max seq no on upload to and download from remote translog

Signed-off-by: Sachin Kale 

* Change log level to debug

Signed-off-by: Sachin Kale 

* Spotless fixes

Signed-off-by: Sachin Kale 

* Fix UTs

Signed-off-by: Sachin Kale 

---------

Signed-off-by: Sachin Kale 
Co-authored-by: Sachin Kale 
---
 .../index/translog/RemoteFsTranslog.java      | 31 +++++++++++++------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
index a305a774f5854..65d16e213cad1 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -103,6 +103,7 @@ public RemoteFsTranslog(
         try {
             download(translogTransferManager, location, logger);
             Checkpoint checkpoint = readCheckpoint(location);
+            logger.info("Downloaded data from remote translog till maxSeqNo = {}", checkpoint.maxSeqNo);
             this.readers.addAll(recoverFromFiles(checkpoint));
             if (readers.isEmpty()) {
                 String errorMsg = String.format(Locale.ROOT, "%s at least one reader must be recovered", shardId);
@@ -266,9 +267,13 @@ public void rollGeneration() throws IOException {
     }
 
     private boolean prepareAndUpload(Long primaryTerm, Long generation) throws IOException {
+        long maxSeqNo = -1;
         try (Releasable ignored = writeLock.acquire()) {
             if (generation == null || generation == current.getGeneration()) {
                 try {
+                    if (closed.get() == false) {
+                        maxSeqNo = getMaxSeqNo();
+                    }
                     final TranslogReader reader = current.closeIntoReader();
                     readers.add(reader);
                     copyCheckpointTo(location.resolve(getCommitCheckpointFileName(current.getGeneration())));
@@ -300,17 +305,17 @@ private boolean prepareAndUpload(Long primaryTerm, Long generation) throws IOExc
             // is not updated in remote translog except in primary to primary recovery.
             if (generation == null) {
                 if (closed.get() == false) {
-                    return upload(primaryTerm, current.getGeneration() - 1);
+                    return upload(primaryTerm, current.getGeneration() - 1, maxSeqNo);
                 } else {
-                    return upload(primaryTerm, current.getGeneration());
+                    return upload(primaryTerm, current.getGeneration(), maxSeqNo);
                 }
             } else {
-                return upload(primaryTerm, generation);
+                return upload(primaryTerm, generation, maxSeqNo);
             }
         }
     }
 
-    private boolean upload(Long primaryTerm, Long generation) throws IOException {
+    private boolean upload(long primaryTerm, long generation, long maxSeqNo) throws IOException {
         // During primary relocation (primary-primary peer recovery), both the old and the new primary have engine
         // created with the RemoteFsTranslog. Both primaries are equipped to upload the translogs. The primary mode check
         // below ensures that the real primary only is uploading. Before the primary mode is set as true for the new
@@ -334,7 +339,7 @@ private boolean upload(Long primaryTerm, Long generation) throws IOException {
         ) {
             return translogTransferManager.transferSnapshot(
                 transferSnapshotProvider,
-                new RemoteFsTranslogTransferListener(generation, primaryTerm)
+                new RemoteFsTranslogTransferListener(generation, primaryTerm, maxSeqNo)
             );
         }
 
@@ -522,23 +527,31 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen
         /**
          * Generation for the translog
          */
-        private final Long generation;
+        private final long generation;
 
         /**
          * Primary Term for the translog
          */
-        private final Long primaryTerm;
+        private final long primaryTerm;
+
+        private final long maxSeqNo;
 
-        RemoteFsTranslogTransferListener(Long generation, Long primaryTerm) {
+        RemoteFsTranslogTransferListener(long generation, long primaryTerm, long maxSeqNo) {
             this.generation = generation;
             this.primaryTerm = primaryTerm;
+            this.maxSeqNo = maxSeqNo;
         }
 
         @Override
         public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOException {
             maxRemoteTranslogGenerationUploaded = generation;
             minRemoteGenReferenced = getMinFileGeneration();
-            logger.trace("uploaded translog for {} {} ", primaryTerm, generation);
+            logger.debug(
+                "Successfully uploaded translog for primary term = {}, generation = {}, maxSeqNo = {}",
+                primaryTerm,
+                generation,
+                maxSeqNo
+            );
         }
 
         @Override

From 448635f77855108afedfe6f2e5c07a2f6c37746c Mon Sep 17 00:00:00 2001
From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com>
Date: Mon, 30 Oct 2023 20:51:16 +0530
Subject: [PATCH 255/388] Disable sort optimization for HALF_FLOAT (#10999)

Signed-off-by: Chaitanya Gohel 
---
 CHANGELOG.md                                                    | 1 +
 .../org/opensearch/index/fielddata/IndexNumericFieldData.java   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 020fb5bda8b8b..c18ff830f84cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -122,6 +122,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
+- [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
 
 ### Deprecated
 
diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
index b4e90b8ab570a..6fc074fe0de95 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
@@ -242,7 +242,7 @@ private XFieldComparatorSource comparatorSource(
                 assert !targetNumericType.isFloatingPoint();
                 source = new IntValuesComparatorSource(this, missingValue, sortMode, nested);
         }
-        if (targetNumericType != getNumericType()) {
+        if (targetNumericType != getNumericType() || getNumericType() == NumericType.HALF_FLOAT) {
             source.disableSkipping(); // disable skipping logic for cast of sort field
         }
         return source;

From b147dc1815aaa039365c98932b4c9859a96d70a3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 30 Oct 2023 12:50:42 -0400
Subject: [PATCH 256/388] Bump org.apache.logging.log4j:log4j-core from 2.21.0
 to 2.21.1 in /buildSrc/src/testKit/thirdPartyAudit/sample_jars (#11000)

* Bump org.apache.logging.log4j:log4j-core

Bumps org.apache.logging.log4j:log4j-core from 2.21.0 to 2.21.1.

---
updated-dependencies:
- dependency-name: org.apache.logging.log4j:log4j-core
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 CHANGELOG.md                                                  | 4 ++--
 buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c18ff830f84cc..e388364893345 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -112,7 +112,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
-- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.0 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.1 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858), [#11000](https://github.com/opensearch-project/OpenSearch/pull/11000))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -138,4 +138,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file
diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
index 74c88e0961c9c..dca2bce94ea6d 100644
--- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
+++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.21.0"
+  implementation "org.apache.logging.log4j:log4j-core:2.21.1"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

From da011ba7ec35770bf4aabf1d26f8e257946fc28f Mon Sep 17 00:00:00 2001
From: Michael Froh 
Date: Mon, 30 Oct 2023 10:09:07 -0700
Subject: [PATCH 257/388] Make search pipelines asynchronous (#10598)

* Make search pipelines asynchronous

If a search processor needs to make a call out to another
service, we should not risk blocking on the transport thread. We should
support async execution.

Signed-off-by: Michael Froh 

* Compute pipelineStart before building request callback chain

Also, IntelliJ suggested refactoring creation of the terminal request
callback into a separate method since the existing method was really
big. I liked that suggestion.

Signed-off-by: Michael Froh 

* Rename async methods (put async at end)

Signed-off-by: Michael Froh 

---------

Signed-off-by: Michael Froh 
---
 CHANGELOG.md                                  |   1 +
 .../action/search/TransportSearchAction.java  |  58 +++--
 .../opensearch/search/pipeline/Pipeline.java  | 206 +++++++++++-------
 .../search/pipeline/PipelinedRequest.java     |   9 +-
 .../pipeline/SearchPipelineService.java       |   3 +-
 .../pipeline/SearchRequestProcessor.java      |  27 +++
 .../pipeline/SearchResponseProcessor.java     |  27 +++
 .../pipeline/SearchPipelineServiceTests.java  |  84 +++++--
 8 files changed, 291 insertions(+), 124 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e388364893345..93b4e4e263063 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -118,6 +118,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
 - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
 - Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
+- Search pipelines now support asynchronous request and response processors to avoid blocking on a transport thread ([#10598](https://github.com/opensearch-project/OpenSearch/pull/10598))
 - [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
 - [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
index a6fb8453af4ff..16b7e4810b130 100644
--- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
@@ -506,24 +506,51 @@ private void executeRequest(
         ActionListener listener;
         try {
             searchRequest = searchPipelineService.resolvePipeline(originalSearchRequest);
-            listener = ActionListener.wrap(
-                r -> originalListener.onResponse(searchRequest.transformResponse(r)),
-                originalListener::onFailure
-            );
+            listener = searchRequest.transformResponseListener(originalListener);
         } catch (Exception e) {
             originalListener.onFailure(e);
             return;
         }
 
-        if (searchQueryMetricsEnabled) {
-            try {
-                searchQueryCategorizer.categorize(searchRequest.source());
-            } catch (Exception e) {
-                logger.error("Error while trying to categorize the query.", e);
+        ActionListener requestTransformListener = ActionListener.wrap(sr -> {
+            if (searchQueryMetricsEnabled) {
+                try {
+                    searchQueryCategorizer.categorize(sr.source());
+                } catch (Exception e) {
+                    logger.error("Error while trying to categorize the query.", e);
+                }
             }
-        }
 
-        ActionListener rewriteListener = ActionListener.wrap(source -> {
+            ActionListener rewriteListener = buildRewriteListener(
+                sr,
+                task,
+                timeProvider,
+                searchAsyncActionProvider,
+                listener,
+                searchRequestOperationsListener
+            );
+            if (sr.source() == null) {
+                rewriteListener.onResponse(sr.source());
+            } else {
+                Rewriteable.rewriteAndFetch(
+                    sr.source(),
+                    searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis),
+                    rewriteListener
+                );
+            }
+        }, listener::onFailure);
+        searchRequest.transformRequest(requestTransformListener);
+    }
+
+    private ActionListener buildRewriteListener(
+        SearchRequest searchRequest,
+        Task task,
+        SearchTimeProvider timeProvider,
+        SearchAsyncActionProvider searchAsyncActionProvider,
+        ActionListener listener,
+        SearchRequestOperationsListener searchRequestOperationsListener
+    ) {
+        return ActionListener.wrap(source -> {
             if (source != searchRequest.source()) {
                 // only set it if it changed - we don't allow null values to be set but it might be already null. this way we catch
                 // situations when source is rewritten to null due to a bug
@@ -634,15 +661,6 @@ private void executeRequest(
                 }
             }
         }, listener::onFailure);
-        if (searchRequest.source() == null) {
-            rewriteListener.onResponse(searchRequest.source());
-        } else {
-            Rewriteable.rewriteAndFetch(
-                searchRequest.source(),
-                searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis),
-                rewriteListener
-            );
-        }
     }
 
     static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) {
diff --git a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java
index d4292b85b20a5..8bab961423f91 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/Pipeline.java
@@ -16,11 +16,13 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.search.SearchPhaseResult;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -117,92 +119,138 @@ protected void afterResponseProcessor(Processor processor, long timeInNanos) {}
 
     protected void onResponseProcessorFailed(Processor processor) {}
 
-    SearchRequest transformRequest(SearchRequest request) throws SearchPipelineProcessingException {
-        if (searchRequestProcessors.isEmpty() == false) {
-            long pipelineStart = relativeTimeSupplier.getAsLong();
-            beforeTransformRequest();
-            try {
-                try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
-                    request.writeTo(bytesStreamOutput);
-                    try (StreamInput in = bytesStreamOutput.bytes().streamInput()) {
-                        try (StreamInput input = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry)) {
-                            request = new SearchRequest(input);
-                        }
-                    }
-                }
-                for (SearchRequestProcessor processor : searchRequestProcessors) {
-                    beforeRequestProcessor(processor);
-                    long start = relativeTimeSupplier.getAsLong();
-                    try {
-                        request = processor.processRequest(request);
-                    } catch (Exception e) {
-                        onRequestProcessorFailed(processor);
-                        if (processor.isIgnoreFailure()) {
-                            logger.warn(
-                                "The exception from request processor ["
-                                    + processor.getType()
-                                    + "] in the search pipeline ["
-                                    + id
-                                    + "] was ignored",
-                                e
-                            );
-                        } else {
-                            throw e;
-                        }
-                    } finally {
-                        long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start);
-                        afterRequestProcessor(processor, took);
-                    }
+    void transformRequest(SearchRequest request, ActionListener requestListener) throws SearchPipelineProcessingException {
+        if (searchRequestProcessors.isEmpty()) {
+            requestListener.onResponse(request);
+            return;
+        }
+
+        try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
+            request.writeTo(bytesStreamOutput);
+            try (StreamInput in = bytesStreamOutput.bytes().streamInput()) {
+                try (StreamInput input = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry)) {
+                    request = new SearchRequest(input);
                 }
-            } catch (Exception e) {
-                onTransformRequestFailure();
-                throw new SearchPipelineProcessingException(e);
-            } finally {
-                long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart);
-                afterTransformRequest(took);
             }
+        } catch (IOException e) {
+            requestListener.onFailure(new SearchPipelineProcessingException(e));
+            return;
         }
-        return request;
+
+        ActionListener finalListener = getTerminalSearchRequestActionListener(requestListener);
+
+        // Chain listeners back-to-front
+        ActionListener currentListener = finalListener;
+        for (int i = searchRequestProcessors.size() - 1; i >= 0; i--) {
+            final ActionListener nextListener = currentListener;
+            SearchRequestProcessor processor = searchRequestProcessors.get(i);
+            currentListener = ActionListener.wrap(r -> {
+                long start = relativeTimeSupplier.getAsLong();
+                beforeRequestProcessor(processor);
+                processor.processRequestAsync(r, ActionListener.wrap(rr -> {
+                    long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start);
+                    afterRequestProcessor(processor, took);
+                    nextListener.onResponse(rr);
+                }, e -> {
+                    long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start);
+                    afterRequestProcessor(processor, took);
+                    onRequestProcessorFailed(processor);
+                    if (processor.isIgnoreFailure()) {
+                        logger.warn(
+                            "The exception from request processor ["
+                                + processor.getType()
+                                + "] in the search pipeline ["
+                                + id
+                                + "] was ignored",
+                            e
+                        );
+                        nextListener.onResponse(r);
+                    } else {
+                        nextListener.onFailure(new SearchPipelineProcessingException(e));
+                    }
+                }));
+            }, finalListener::onFailure);
+        }
+
+        beforeTransformRequest();
+        currentListener.onResponse(request);
     }
 
-    SearchResponse transformResponse(SearchRequest request, SearchResponse response) throws SearchPipelineProcessingException {
-        if (searchResponseProcessors.isEmpty() == false) {
-            long pipelineStart = relativeTimeSupplier.getAsLong();
-            beforeTransformResponse();
-            try {
-                for (SearchResponseProcessor processor : searchResponseProcessors) {
-                    beforeResponseProcessor(processor);
-                    long start = relativeTimeSupplier.getAsLong();
-                    try {
-                        response = processor.processResponse(request, response);
-                    } catch (Exception e) {
-                        onResponseProcessorFailed(processor);
-                        if (processor.isIgnoreFailure()) {
-                            logger.warn(
-                                "The exception from response processor ["
-                                    + processor.getType()
-                                    + "] in the search pipeline ["
-                                    + id
-                                    + "] was ignored",
-                                e
-                            );
-                        } else {
-                            throw e;
-                        }
-                    } finally {
-                        long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start);
-                        afterResponseProcessor(processor, took);
+    private ActionListener getTerminalSearchRequestActionListener(ActionListener requestListener) {
+        final long pipelineStart = relativeTimeSupplier.getAsLong();
+
+        return ActionListener.wrap(r -> {
+            long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart);
+            afterTransformRequest(took);
+            requestListener.onResponse(new PipelinedRequest(this, r));
+        }, e -> {
+            long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart);
+            afterTransformRequest(took);
+            onTransformRequestFailure();
+            requestListener.onFailure(new SearchPipelineProcessingException(e));
+        });
+    }
+
+    ActionListener transformResponseListener(SearchRequest request, ActionListener responseListener) {
+        if (searchResponseProcessors.isEmpty()) {
+            // No response transformation necessary
+            return responseListener;
+        }
+
+        long[] pipelineStart = new long[1];
+
+        final ActionListener originalListener = responseListener;
+        responseListener = ActionListener.wrap(r -> {
+            long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart[0]);
+            afterTransformResponse(took);
+            originalListener.onResponse(r);
+        }, e -> {
+            long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart[0]);
+            afterTransformResponse(took);
+            onTransformResponseFailure();
+            originalListener.onFailure(e);
+        });
+        ActionListener finalListener = responseListener; // Jump directly to this one on exception.
+
+        for (int i = searchResponseProcessors.size() - 1; i >= 0; i--) {
+            final ActionListener currentFinalListener = responseListener;
+            final SearchResponseProcessor processor = searchResponseProcessors.get(i);
+
+            responseListener = ActionListener.wrap(r -> {
+                beforeResponseProcessor(processor);
+                final long start = relativeTimeSupplier.getAsLong();
+                processor.processResponseAsync(request, r, ActionListener.wrap(rr -> {
+                    long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start);
+                    afterResponseProcessor(processor, took);
+                    currentFinalListener.onResponse(rr);
+                }, e -> {
+                    onResponseProcessorFailed(processor);
+                    long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - start);
+                    afterResponseProcessor(processor, took);
+                    if (processor.isIgnoreFailure()) {
+                        logger.warn(
+                            "The exception from response processor ["
+                                + processor.getType()
+                                + "] in the search pipeline ["
+                                + id
+                                + "] was ignored",
+                            e
+                        );
+                        // Pass the previous response through to the next processor in the chain
+                        currentFinalListener.onResponse(r);
+                    } else {
+                        currentFinalListener.onFailure(new SearchPipelineProcessingException(e));
                     }
-                }
-            } catch (Exception e) {
-                onTransformResponseFailure();
-                throw new SearchPipelineProcessingException(e);
-            } finally {
-                long took = TimeUnit.NANOSECONDS.toMillis(relativeTimeSupplier.getAsLong() - pipelineStart);
-                afterTransformResponse(took);
-            }
+                }));
+            }, finalListener::onFailure);
         }
-        return response;
+        final ActionListener chainListener = responseListener;
+        return ActionListener.wrap(r -> {
+            beforeTransformResponse();
+            pipelineStart[0] = relativeTimeSupplier.getAsLong();
+            chainListener.onResponse(r);
+        }, originalListener::onFailure);
+
     }
 
      void runSearchPhaseResultsTransformer(
diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java b/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java
index 5a7539808c127..77dfc6bcd4fc5 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java
@@ -12,6 +12,7 @@
 import org.opensearch.action.search.SearchPhaseResults;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.search.SearchPhaseResult;
 
 /**
@@ -27,8 +28,12 @@ public final class PipelinedRequest extends SearchRequest {
         this.pipeline = pipeline;
     }
 
-    public SearchResponse transformResponse(SearchResponse response) {
-        return pipeline.transformResponse(this, response);
+    public void transformRequest(ActionListener requestListener) {
+        pipeline.transformRequest(this, requestListener);
+    }
+
+    public ActionListener transformResponseListener(ActionListener responseListener) {
+        return pipeline.transformResponseListener(this, responseListener);
     }
 
     public  void transformSearchPhaseResults(
diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
index 739101519ff98..580fe1b7c4216 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
@@ -408,8 +408,7 @@ public PipelinedRequest resolvePipeline(SearchRequest searchRequest) {
                 pipeline = pipelineHolder.pipeline;
             }
         }
-        SearchRequest transformedRequest = pipeline.transformRequest(searchRequest);
-        return new PipelinedRequest(pipeline, transformedRequest);
+        return new PipelinedRequest(pipeline, searchRequest);
     }
 
     Map> getRequestProcessorFactories() {
diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchRequestProcessor.java b/server/src/main/java/org/opensearch/search/pipeline/SearchRequestProcessor.java
index c236cde1a5cc0..427c9e4ab694c 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/SearchRequestProcessor.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/SearchRequestProcessor.java
@@ -9,10 +9,37 @@
 package org.opensearch.search.pipeline;
 
 import org.opensearch.action.search.SearchRequest;
+import org.opensearch.core.action.ActionListener;
 
 /**
  * Interface for a search pipeline processor that modifies a search request.
  */
 public interface SearchRequestProcessor extends Processor {
+
+    /**
+     * Transform a {@link SearchRequest}. Executed on the coordinator node before any {@link org.opensearch.action.search.SearchPhase}
+     * executes.
+     * 
                              
+     * Implement this method if the processor makes no asynchronous calls.
+     * @param request the executed {@link SearchRequest}
+     * @return a new {@link SearchRequest} (or the input {@link SearchRequest} if no changes)
+     * @throws Exception if an error occurs during processing
+     */
     SearchRequest processRequest(SearchRequest request) throws Exception;
+
+    /**
+     * Transform a {@link SearchRequest}. Executed on the coordinator node before any {@link org.opensearch.action.search.SearchPhase}
+     * executes.
+     * 
                              
+     * Expert method: Implement this if the processor needs to make asynchronous calls. Otherwise, implement processRequest.
+     * @param request the executed {@link SearchRequest}
+     * @param requestListener callback to be invoked on successful processing or on failure
+     */
+    default void processRequestAsync(SearchRequest request, ActionListener requestListener) {
+        try {
+            requestListener.onResponse(processRequest(request));
+        } catch (Exception e) {
+            requestListener.onFailure(e);
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchResponseProcessor.java b/server/src/main/java/org/opensearch/search/pipeline/SearchResponseProcessor.java
index 2f22cedb9b5c0..21136ce208fee 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/SearchResponseProcessor.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/SearchResponseProcessor.java
@@ -10,10 +10,37 @@
 
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
+import org.opensearch.core.action.ActionListener;
 
 /**
  * Interface for a search pipeline processor that modifies a search response.
  */
 public interface SearchResponseProcessor extends Processor {
+
+    /**
+     * Transform a {@link SearchResponse}, possibly based on the executed {@link SearchRequest}.
+     * 
                              
+     * Implement this method if the processor makes no asynchronous calls.
+     * @param request the executed {@link SearchRequest}
+     * @param response the current {@link SearchResponse}, possibly modified by earlier processors
+     * @return a modified {@link SearchResponse} (or the input {@link SearchResponse} if no changes)
+     * @throws Exception if an error occurs during processing
+     */
     SearchResponse processResponse(SearchRequest request, SearchResponse response) throws Exception;
+
+    /**
+     * Transform a {@link SearchResponse}, possibly based on the executed {@link SearchRequest}.
+     * 
                              
+     * Expert method: Implement this if the processor needs to make asynchronous calls. Otherwise, implement processResponse.
+     * @param request the executed {@link SearchRequest}
+     * @param response the current {@link SearchResponse}, possibly modified by earlier processors
+     * @param responseListener callback to be invoked on successful processing or on failure
+     */
+    default void processResponseAsync(SearchRequest request, SearchResponse response, ActionListener responseListener) {
+        try {
+            responseListener.onResponse(processResponse(request, response));
+        } catch (Exception e) {
+            responseListener.onFailure(e);
+        }
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
index d44bd3831281f..98d2a7e84d672 100644
--- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
@@ -41,6 +41,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.concurrent.AtomicArray;
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.breaker.CircuitBreaker;
 import org.opensearch.core.common.breaker.NoopCircuitBreaker;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -194,7 +195,7 @@ public void testResolveIndexDefaultPipeline() throws Exception {
         service.applyClusterState(cce);
 
         SearchRequest searchRequest = new SearchRequest("my_index").source(SearchSourceBuilder.searchSource().size(5));
-        PipelinedRequest pipelinedRequest = service.resolvePipeline(searchRequest);
+        PipelinedRequest pipelinedRequest = syncTransformRequest(service.resolvePipeline(searchRequest));
         assertEquals("p1", pipelinedRequest.getPipeline().getId());
         assertEquals(10, pipelinedRequest.source().size());
 
@@ -597,7 +598,7 @@ public void testTransformRequest() throws Exception {
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(new TermQueryBuilder("foo", "bar")).size(size);
         SearchRequest request = new SearchRequest("_index").source(sourceBuilder).pipeline("p1");
 
-        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(request);
+        PipelinedRequest pipelinedRequest = syncTransformRequest(searchPipelineService.resolvePipeline(request));
 
         assertEquals(2 * size, pipelinedRequest.source().size());
         assertEquals(size, request.source().size());
@@ -641,19 +642,57 @@ public void testTransformResponse() throws Exception {
         // First try without specifying a pipeline, which should be a no-op.
         SearchRequest searchRequest = new SearchRequest();
         PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
-        SearchResponse notTransformedResponse = pipelinedRequest.transformResponse(searchResponse);
+        SearchResponse notTransformedResponse = syncTransformResponse(pipelinedRequest, searchResponse);
         assertSame(searchResponse, notTransformedResponse);
 
         // Now apply a pipeline
         searchRequest = new SearchRequest().pipeline("p1");
         pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
-        SearchResponse transformedResponse = pipelinedRequest.transformResponse(searchResponse);
+        SearchResponse transformedResponse = syncTransformResponse(pipelinedRequest, searchResponse);
         assertEquals(size, transformedResponse.getHits().getHits().length);
         for (int i = 0; i < size; i++) {
             assertEquals(2.0, transformedResponse.getHits().getHits()[i].getScore(), 0.0001f);
         }
     }
 
+    /**
+     * Helper to synchronously apply a response pipeline, returning the transformed response.
+     */
+    private static SearchResponse syncTransformResponse(PipelinedRequest pipelinedRequest, SearchResponse searchResponse) throws Exception {
+        SearchResponse[] responseBox = new SearchResponse[1];
+        Exception[] exceptionBox = new Exception[1];
+        ActionListener responseListener = pipelinedRequest.transformResponseListener(ActionListener.wrap(r -> {
+            responseBox[0] = r;
+        }, e -> { exceptionBox[0] = e; }));
+        responseListener.onResponse(searchResponse);
+
+        if (exceptionBox[0] != null) {
+            throw exceptionBox[0];
+        }
+        return responseBox[0];
+    }
+
+    /**
+     * Helper to synchronously apply a request pipeline, returning the transformed request.
+     */
+    private static PipelinedRequest syncTransformRequest(PipelinedRequest request) throws Exception {
+        PipelinedRequest[] requestBox = new PipelinedRequest[1];
+        Exception[] exceptionBox = new Exception[1];
+
+        request.transformRequest(ActionListener.wrap(r -> requestBox[0] = (PipelinedRequest) r, e -> exceptionBox[0] = e));
+        if (exceptionBox[0] != null) {
+            throw exceptionBox[0];
+        }
+        return requestBox[0];
+    }
+
+    /**
+     * Helper to synchronously apply a request pipeline and response pipeline, returning the transformed response.
+     */
+    private static SearchResponse syncExecutePipeline(PipelinedRequest request, SearchResponse response) throws Exception {
+        return syncTransformResponse(syncTransformRequest(request), response);
+    }
+
     public void testTransformSearchPhase() {
         SearchPipelineService searchPipelineService = createWithProcessors();
         SearchPipelineMetadata metadata = new SearchPipelineMetadata(
@@ -875,7 +914,7 @@ public void testInlinePipeline() throws Exception {
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
 
         // Verify pipeline
-        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        PipelinedRequest pipelinedRequest = syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest));
         Pipeline pipeline = pipelinedRequest.getPipeline();
         assertEquals(SearchPipelineService.AD_HOC_PIPELINE_ID, pipeline.getId());
         assertEquals(1, pipeline.getSearchRequestProcessors().size());
@@ -894,7 +933,7 @@ public void testInlinePipeline() throws Exception {
         SearchResponseSections searchResponseSections = new SearchResponseSections(searchHits, null, null, false, false, null, 0);
         SearchResponse searchResponse = new SearchResponse(searchResponseSections, null, 1, 1, 0, 10, null, null);
 
-        SearchResponse transformedResponse = pipeline.transformResponse(searchRequest, searchResponse);
+        SearchResponse transformedResponse = syncTransformResponse(pipelinedRequest, searchResponse);
         for (int i = 0; i < size; i++) {
             assertEquals(2.0, transformedResponse.getHits().getHits()[i].getScore(), 0.0001);
         }
@@ -946,7 +985,10 @@ public void testExceptionOnRequestProcessing() {
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
 
         // Exception thrown when processing the request
-        expectThrows(SearchPipelineProcessingException.class, () -> searchPipelineService.resolvePipeline(searchRequest));
+        expectThrows(
+            SearchPipelineProcessingException.class,
+            () -> syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest))
+        );
     }
 
     public void testExceptionOnResponseProcessing() throws Exception {
@@ -974,10 +1016,10 @@ public void testExceptionOnResponseProcessing() throws Exception {
 
         SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null);
         // Exception thrown when processing response
-        expectThrows(SearchPipelineProcessingException.class, () -> pipelinedRequest.transformResponse(response));
+        expectThrows(SearchPipelineProcessingException.class, () -> syncTransformResponse(pipelinedRequest, response));
     }
 
-    public void testCatchExceptionOnRequestProcessing() throws IllegalAccessException {
+    public void testCatchExceptionOnRequestProcessing() throws Exception {
         SearchRequestProcessor throwingRequestProcessor = new FakeRequestProcessor("throwing_request", null, null, true, r -> {
             throw new RuntimeException();
         });
@@ -1008,7 +1050,7 @@ public void testCatchExceptionOnRequestProcessing() throws IllegalAccessExceptio
                     "The exception from request processor [throwing_request] in the search pipeline [_ad_hoc_pipeline] was ignored"
                 )
             );
-            PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+            syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest));
             mockAppender.assertAllExpectationsMatched();
         }
     }
@@ -1048,7 +1090,7 @@ public void testCatchExceptionOnResponseProcessing() throws Exception {
                     "The exception from response processor [throwing_response] in the search pipeline [_ad_hoc_pipeline] was ignored"
                 )
             );
-            pipelinedRequest.transformResponse(response);
+            syncTransformResponse(pipelinedRequest, response);
             mockAppender.assertAllExpectationsMatched();
         }
     }
@@ -1078,15 +1120,15 @@ public void testStats() throws Exception {
         SearchRequest request = new SearchRequest();
         SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null);
 
-        searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")).transformResponse(response);
+        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")), response);
         expectThrows(
             SearchPipelineProcessingException.class,
-            () -> searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")).transformResponse(response)
+            () -> syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")), response)
         );
-        searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")).transformResponse(response);
+        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")), response);
         expectThrows(
             SearchPipelineProcessingException.class,
-            () -> searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")).transformResponse(response)
+            () -> syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")), response)
         );
 
         SearchPipelineStats stats = searchPipelineService.stats();
@@ -1164,12 +1206,12 @@ public void testStatsEnabledIgnoreFailure() throws Exception {
         SearchRequest request = new SearchRequest();
         SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null);
 
-        searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")).transformResponse(response);
+        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")), response);
         // Caught Exception here
-        searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")).transformResponse(response);
-        searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")).transformResponse(response);
+        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")), response);
+        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")), response);
         // Caught Exception here
-        searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")).transformResponse(response);
+        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")), response);
 
         // when ignoreFailure enabled, the search pipelines will all succeed.
         SearchPipelineStats stats = searchPipelineService.stats();
@@ -1273,8 +1315,8 @@ private SearchPipelineService getSearchPipelineService(
     }
 
     private static void assertPipelineStats(OperationStats stats, long count, long failed) {
-        assertEquals(stats.getCount(), count);
-        assertEquals(stats.getFailedCount(), failed);
+        assertEquals(count, stats.getCount());
+        assertEquals(failed, stats.getFailedCount());
     }
 
     public void testAdHocRejectingProcessor() {

From 9d85e566894ef53e5f2093618b3d455e4d0a04ce Mon Sep 17 00:00:00 2001
From: gaobinlong 
Date: Tue, 31 Oct 2023 02:31:36 +0800
Subject: [PATCH 258/388] Fix passing wrong parameter when calling
 newConfigurationException() in DotExpanderProcessor (#10737)

* Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor

Signed-off-by: Gao Binlong 

* modify change log

Signed-off-by: Gao Binlong 

---------

Signed-off-by: Gao Binlong 
---
 CHANGELOG.md                                     |  1 +
 .../ingest/common/DotExpanderProcessor.java      | 16 +++-------------
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 93b4e4e263063..1d0c30d9812c3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -135,6 +135,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
 - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
 - Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
+- Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
 
 ### Security
 
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java
index 39c2d67ac0b85..0eab6334854ab 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/DotExpanderProcessor.java
@@ -118,25 +118,15 @@ public Processor create(
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
             if (field.contains(".") == false) {
-                throw ConfigurationUtils.newConfigurationException(
-                    ConfigurationUtils.TAG_KEY,
-                    tag,
-                    "field",
-                    "field does not contain a dot"
-                );
+                throw ConfigurationUtils.newConfigurationException(TYPE, tag, "field", "field does not contain a dot");
             }
             if (field.indexOf('.') == 0 || field.lastIndexOf('.') == field.length() - 1) {
-                throw ConfigurationUtils.newConfigurationException(
-                    ConfigurationUtils.TAG_KEY,
-                    tag,
-                    "field",
-                    "Field can't start or end with a dot"
-                );
+                throw ConfigurationUtils.newConfigurationException(TYPE, tag, "field", "Field can't start or end with a dot");
             }
             int firstIndex = -1;
             for (int index = field.indexOf('.'); index != -1; index = field.indexOf('.', index + 1)) {
                 if (index - firstIndex == 1) {
-                    throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field", "No space between dots");
+                    throw ConfigurationUtils.newConfigurationException(TYPE, tag, "field", "No space between dots");
                 }
                 firstIndex = index;
             }

From 9c65350481e1ec14c00fee4cccf6e92c98da0d2c Mon Sep 17 00:00:00 2001
From: Suraj Singh 
Date: Mon, 30 Oct 2023 19:09:31 -0700
Subject: [PATCH 259/388] [Segment Replication] Remove primary targets from
 replication tracker (#11011)

Signed-off-by: Suraj Singh 
---
 .../index/seqno/ReplicationTracker.java       | 12 +++
 .../index/seqno/ReplicationTrackerTests.java  | 80 +++++++++++++++++++
 2 files changed, 92 insertions(+)

diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
index 94f376d923689..352167597fa81 100644
--- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
@@ -71,6 +71,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
@@ -1227,6 +1228,14 @@ public ReplicationCheckpoint getLatestReplicationCheckpoint() {
         return this.latestReplicationCheckpoint;
     }
 
+    private boolean isPrimaryRelocation(String allocationId) {
+        Optional shardRouting = routingTable.shards()
+            .stream()
+            .filter(routing -> routing.allocationId().getId().equals(allocationId))
+            .findAny();
+        return shardRouting.isPresent() && shardRouting.get().primary();
+    }
+
     private void createReplicationLagTimers() {
         for (Map.Entry entry : checkpoints.entrySet()) {
             final String allocationId = entry.getKey();
@@ -1236,6 +1245,7 @@ private void createReplicationLagTimers() {
                 // it is possible for a shard to be in-sync but not yet removed from the checkpoints collection after a failover event.
                 if (cps.inSync
                     && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false
+                    && isPrimaryRelocation(allocationId) == false
                     && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint)) {
                     cps.checkpointTimers.computeIfAbsent(latestReplicationCheckpoint, ignored -> new SegmentReplicationLagTimer());
                     logger.trace(
@@ -1267,6 +1277,7 @@ public synchronized void startReplicationLagTimers(ReplicationCheckpoint checkpo
                 final CheckpointState cps = e.getValue();
                 if (cps.inSync
                     && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false
+                    && isPrimaryRelocation(e.getKey()) == false
                     && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint)
                     && cps.checkpointTimers.containsKey(latestReplicationCheckpoint)) {
                     cps.checkpointTimers.get(latestReplicationCheckpoint).start();
@@ -1291,6 +1302,7 @@ public synchronized Set getSegmentReplicationStats
                     entry -> entry.getKey().equals(this.shardAllocationId) == false
                         && entry.getValue().inSync
                         && replicationGroup.getUnavailableInSyncShards().contains(entry.getKey()) == false
+                        && isPrimaryRelocation(entry.getKey()) == false
                 )
                 .map(entry -> buildShardStats(entry.getKey(), entry.getValue()))
                 .collect(Collectors.toUnmodifiableSet());
diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
index 28c95ddf13fc4..7971591e82bab 100644
--- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
@@ -1907,6 +1907,86 @@ public void testSegmentReplicationCheckpointTracking() {
         }
     }
 
+    public void testSegmentReplicationCheckpointForRelocatingPrimary() {
+        Settings settings = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build();
+        final long initialClusterStateVersion = randomNonNegativeLong();
+        final int numberOfActiveAllocationsIds = randomIntBetween(2, 2);
+        final int numberOfInitializingIds = randomIntBetween(2, 2);
+        final Tuple, Set> activeAndInitializingAllocationIds = randomActiveAndInitializingAllocationIds(
+            numberOfActiveAllocationsIds,
+            numberOfInitializingIds
+        );
+        final Set activeAllocationIds = activeAndInitializingAllocationIds.v1();
+        final Set initializingIds = activeAndInitializingAllocationIds.v2();
+
+        AllocationId targetAllocationId = initializingIds.iterator().next();
+        AllocationId primaryId = activeAllocationIds.iterator().next();
+        String relocatingToNodeId = nodeIdFromAllocationId(targetAllocationId);
+
+        logger.info("--> activeAllocationIds {} Primary {}", activeAllocationIds, primaryId.getId());
+        logger.info("--> initializingIds {} Target {}", initializingIds, targetAllocationId);
+
+        final ShardId shardId = new ShardId("test", "_na_", 0);
+        final IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(shardId);
+        for (final AllocationId initializingId : initializingIds) {
+            boolean primaryRelocationTarget = initializingId.equals(targetAllocationId);
+            builder.addShard(
+                TestShardRouting.newShardRouting(
+                    shardId,
+                    nodeIdFromAllocationId(initializingId),
+                    null,
+                    primaryRelocationTarget,
+                    ShardRoutingState.INITIALIZING,
+                    initializingId
+                )
+            );
+        }
+        builder.addShard(
+            TestShardRouting.newShardRouting(
+                shardId,
+                nodeIdFromAllocationId(primaryId),
+                relocatingToNodeId,
+                true,
+                ShardRoutingState.STARTED,
+                primaryId
+            )
+        );
+        IndexShardRoutingTable routingTable = builder.build();
+        final ReplicationTracker tracker = newTracker(primaryId, settings);
+        tracker.updateFromClusterManager(initialClusterStateVersion, ids(activeAllocationIds), routingTable);
+        tracker.activatePrimaryMode(NO_OPS_PERFORMED);
+        assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds)));
+        assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable));
+        assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
+        initializingIds.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED));
+
+        final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 5L, "abcd", Version.LATEST);
+        final ReplicationCheckpoint initialCheckpoint = new ReplicationCheckpoint(
+            tracker.shardId(),
+            0L,
+            1,
+            1,
+            5L,
+            Codec.getDefault().getName(),
+            Map.of("segment_1", segment_1)
+        );
+        tracker.setLatestReplicationCheckpoint(initialCheckpoint);
+        tracker.startReplicationLagTimers(initialCheckpoint);
+
+        final Set expectedIds = initializingIds.stream()
+            .filter(id -> id.equals(targetAllocationId))
+            .map(AllocationId::getId)
+            .collect(Collectors.toSet());
+
+        Set groupStats = tracker.getSegmentReplicationStats();
+        assertEquals(expectedIds.size(), groupStats.size());
+        for (SegmentReplicationShardStats shardStat : groupStats) {
+            assertEquals(1, shardStat.getCheckpointsBehindCount());
+            assertEquals(5L, shardStat.getBytesBehindCount());
+            assertTrue(shardStat.getCurrentReplicationLagMillis() >= shardStat.getCurrentReplicationTimeMillis());
+        }
+    }
+
     public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() {
         Settings settings = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build();
         final long initialClusterStateVersion = randomNonNegativeLong();

From 63aff16c0d08bac44e1d1a6158ee3f838a043074 Mon Sep 17 00:00:00 2001
From: Harsha Vamsi Kalluri 
Date: Mon, 30 Oct 2023 22:10:36 -0700
Subject: [PATCH 260/388] Adds various Query overrides to Keyword Field
 (#10425)

The keywordfield mapper provides access to various query types, e.g. the termsQuery, fuzzyQuery. These are inherited as is from the StringType. But we do not take into account the fact that keyword fields can have doc_values enabled. This PR adds the ability for various queries to first check if doc_values are enabled and if so out-source the work to lucene to decide if it's better to use index values or doc_values when running queries.

Signed-off-by: Harsha Vamsi Kalluri 
---
 CHANGELOG.md                                  |   1 +
 .../test/search/340_keyword_doc_values.yml    |  46 ++++
 .../index/mapper/KeywordFieldMapper.java      | 238 +++++++++++++++++-
 .../index/mapper/MappedFieldType.java         |  24 ++
 .../subphase/highlight/CustomQueryScorer.java |   3 +
 .../index/mapper/KeywordFieldTypeTests.java   | 144 ++++++++++-
 .../query/MultiMatchQueryBuilderTests.java    |   8 +-
 7 files changed, 446 insertions(+), 18 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/340_keyword_doc_values.yml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1d0c30d9812c3..046693421d07b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -63,6 +63,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855))
 - Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
 - Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558))
+- Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 
 
 ### Deprecated
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_keyword_doc_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_keyword_doc_values.yml
new file mode 100644
index 0000000000000..8829e7b100fdd
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_keyword_doc_values.yml
@@ -0,0 +1,46 @@
+---
+"search on keyword fields with doc_values enabled":
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              "some_keyword":
+                type: "keyword"
+                index: true
+                doc_values: true
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{"index": {"_index": "test", "_id": "1" }}'
+          - '{ "some_keyword": "ingesting some random keyword data" }'
+          - '{ "index": { "_index": "test", "_id": "2" }}'
+          - '{ "some_keyword": "400" }'
+          - '{ "index": { "_index": "test", "_id": "3" } }'
+          - '{ "some_keyword": "5" }'
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            prefix:
+              some_keyword: "ing"
+
+  - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" }
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            range: {
+              "some_keyword": {
+                "lt": 500
+              } }
+
+  - match: { hits.total.value: 2 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
index 92ee8067ee4a0..c14b2c92c89c3 100644
--- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
@@ -38,11 +38,24 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RegexpQuery;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Operations;
+import org.opensearch.OpenSearchException;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.lucene.BytesRefs;
 import org.opensearch.common.lucene.Lucene;
+import org.opensearch.common.lucene.search.AutomatonQueries;
+import org.opensearch.common.unit.Fuzziness;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.analysis.IndexAnalyzers;
 import org.opensearch.index.analysis.NamedAnalyzer;
@@ -62,6 +75,8 @@
 import java.util.Objects;
 import java.util.function.Supplier;
 
+import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
+
 /**
  * A field mapper for keywords. This mapper accepts strings and indexes them as-is.
  *
@@ -317,7 +332,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
         @Override
         public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
             if (format != null) {
-                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't " + "support formats.");
             }
 
             return new SourceValueFetcher(name(), context, nullValue) {
@@ -372,17 +387,226 @@ protected BytesRef indexedValueForSearch(Object value) {
             return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString());
         }
 
+        @Override
+        public Query termsQuery(List values, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
+            // has index and doc_values enabled
+            if (isSearchable() && hasDocValues()) {
+                BytesRef[] bytesRefs = new BytesRef[values.size()];
+                for (int i = 0; i < bytesRefs.length; i++) {
+                    bytesRefs[i] = indexedValueForSearch(values.get(i));
+                }
+                Query indexQuery = new TermInSetQuery(name(), bytesRefs);
+                Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs);
+                return new IndexOrDocValuesQuery(indexQuery, dvQuery);
+            }
+            // if we only have doc_values enabled, we construct a new query with doc_values re-written
+            if (hasDocValues()) {
+                BytesRef[] bytesRefs = new BytesRef[values.size()];
+                for (int i = 0; i < bytesRefs.length; i++) {
+                    bytesRefs[i] = indexedValueForSearch(values.get(i));
+                }
+                return new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs);
+            }
+            // has index enabled, we're going to return the query as is
+            return super.termsQuery(values, context);
+        }
+
+        @Override
+        public Query prefixQuery(
+            String value,
+            @Nullable MultiTermQuery.RewriteMethod method,
+            boolean caseInsensitive,
+            QueryShardContext context
+        ) {
+            if (context.allowExpensiveQueries() == false) {
+                throw new OpenSearchException(
+                    "[prefix] queries cannot be executed when '"
+                        + ALLOW_EXPENSIVE_QUERIES.getKey()
+                        + "' is set to false. For optimised prefix queries on text "
+                        + "fields please enable [index_prefixes]."
+                );
+            }
+            failIfNotIndexedAndNoDocValues();
+            if (isSearchable() && hasDocValues()) {
+                Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context);
+                Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context);
+                return new IndexOrDocValuesQuery(indexQuery, dvQuery);
+            }
+            if (hasDocValues()) {
+                if (caseInsensitive) {
+                    return AutomatonQueries.caseInsensitivePrefixQuery(
+                        (new Term(name(), indexedValueForSearch(value))),
+                        MultiTermQuery.DOC_VALUES_REWRITE
+                    );
+                }
+                return new PrefixQuery(new Term(name(), indexedValueForSearch(value)), MultiTermQuery.DOC_VALUES_REWRITE);
+            }
+            return super.prefixQuery(value, method, caseInsensitive, context);
+        }
+
+        @Override
+        public Query regexpQuery(
+            String value,
+            int syntaxFlags,
+            int matchFlags,
+            int maxDeterminizedStates,
+            @Nullable MultiTermQuery.RewriteMethod method,
+            QueryShardContext context
+        ) {
+            if (context.allowExpensiveQueries() == false) {
+                throw new OpenSearchException(
+                    "[regexp] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to " + "false."
+                );
+            }
+            failIfNotIndexedAndNoDocValues();
+            if (isSearchable() && hasDocValues()) {
+                Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
+                Query dvQuery = super.regexpQuery(
+                    value,
+                    syntaxFlags,
+                    matchFlags,
+                    maxDeterminizedStates,
+                    MultiTermQuery.DOC_VALUES_REWRITE,
+                    context
+                );
+                return new IndexOrDocValuesQuery(indexQuery, dvQuery);
+            }
+            if (hasDocValues()) {
+                return new RegexpQuery(
+                    new Term(name(), indexedValueForSearch(value)),
+                    syntaxFlags,
+                    matchFlags,
+                    RegexpQuery.DEFAULT_PROVIDER,
+                    maxDeterminizedStates,
+                    MultiTermQuery.DOC_VALUES_REWRITE
+                );
+            }
+            return super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
+        }
+
+        @Override
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
+            if (context.allowExpensiveQueries() == false) {
+                throw new OpenSearchException(
+                    "[range] queries on [text] or [keyword] fields cannot be executed when '"
+                        + ALLOW_EXPENSIVE_QUERIES.getKey()
+                        + "' is set to false."
+                );
+            }
+            failIfNotIndexedAndNoDocValues();
+            if (isSearchable() && hasDocValues()) {
+                Query indexQuery = new TermRangeQuery(
+                    name(),
+                    lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
+                    upperTerm == null ? null : indexedValueForSearch(upperTerm),
+                    includeLower,
+                    includeUpper
+                );
+                Query dvQuery = new TermRangeQuery(
+                    name(),
+                    lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
+                    upperTerm == null ? null : indexedValueForSearch(upperTerm),
+                    includeLower,
+                    includeUpper,
+                    MultiTermQuery.DOC_VALUES_REWRITE
+                );
+                return new IndexOrDocValuesQuery(indexQuery, dvQuery);
+            }
+            if (hasDocValues()) {
+                return new TermRangeQuery(
+                    name(),
+                    lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
+                    upperTerm == null ? null : indexedValueForSearch(upperTerm),
+                    includeLower,
+                    includeUpper,
+                    MultiTermQuery.DOC_VALUES_REWRITE
+                );
+            }
+            return new TermRangeQuery(
+                name(),
+                lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
+                upperTerm == null ? null : indexedValueForSearch(upperTerm),
+                includeLower,
+                includeUpper
+            );
+        }
+
+        @Override
+        public Query fuzzyQuery(
+            Object value,
+            Fuzziness fuzziness,
+            int prefixLength,
+            int maxExpansions,
+            boolean transpositions,
+            @Nullable MultiTermQuery.RewriteMethod method,
+            QueryShardContext context
+        ) {
+            failIfNotIndexedAndNoDocValues();
+            if (context.allowExpensiveQueries() == false) {
+                throw new OpenSearchException(
+                    "[fuzzy] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to " + "false."
+                );
+            }
+            if (isSearchable() && hasDocValues()) {
+                Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context);
+                Query dvQuery = super.fuzzyQuery(
+                    value,
+                    fuzziness,
+                    prefixLength,
+                    maxExpansions,
+                    transpositions,
+                    MultiTermQuery.DOC_VALUES_REWRITE,
+                    context
+                );
+                return new IndexOrDocValuesQuery(indexQuery, dvQuery);
+            }
+            if (hasDocValues()) {
+                return new FuzzyQuery(
+                    new Term(name(), indexedValueForSearch(value)),
+                    fuzziness.asDistance(BytesRefs.toString(value)),
+                    prefixLength,
+                    maxExpansions,
+                    transpositions,
+                    MultiTermQuery.DOC_VALUES_REWRITE
+                );
+            }
+            return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context);
+        }
+
         @Override
         public Query wildcardQuery(
             String value,
             @Nullable MultiTermQuery.RewriteMethod method,
-            boolean caseInsensitve,
+            boolean caseInsensitive,
             QueryShardContext context
         ) {
-            // keyword field types are always normalized, so ignore case sensitivity and force normalize the wildcard
+            if (context.allowExpensiveQueries() == false) {
+                throw new OpenSearchException(
+                    "[wildcard] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to " + "false."
+                );
+            }
+            failIfNotIndexedAndNoDocValues();
+            // keyword field types are always normalized, so ignore case sensitivity and force normalize the
+            // wildcard
             // query text
-            return super.wildcardQuery(value, method, caseInsensitve, true, context);
+            if (isSearchable() && hasDocValues()) {
+                Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context);
+                Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context);
+                return new IndexOrDocValuesQuery(indexQuery, dvQuery);
+            }
+            if (hasDocValues()) {
+                Term term;
+                value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer());
+                term = new Term(name(), value);
+                if (caseInsensitive) {
+                    return AutomatonQueries.caseInsensitiveWildcardQuery(term, method);
+                }
+                return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE);
+            }
+            return super.wildcardQuery(value, method, caseInsensitive, true, context);
         }
+
     }
 
     private final boolean indexed;
@@ -422,8 +646,10 @@ protected KeywordFieldMapper(
         this.indexAnalyzers = builder.indexAnalyzers;
     }
 
-    /** Values that have more chars than the return value of this method will
-     *  be skipped at parsing time. */
+    /**
+     * Values that have more chars than the return value of this method will
+     * be skipped at parsing time.
+     */
     public int ignoreAbove() {
         return ignoreAbove;
     }
diff --git a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java
index 997835f712038..62acad99074c2 100644
--- a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java
@@ -269,6 +269,21 @@ public Query fuzzyQuery(
         );
     }
 
+    // Fuzzy Query with re-write method
+    public Query fuzzyQuery(
+        Object value,
+        Fuzziness fuzziness,
+        int prefixLength,
+        int maxExpansions,
+        boolean transpositions,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        QueryShardContext context
+    ) {
+        throw new IllegalArgumentException(
+            "Can only use fuzzy queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]"
+        );
+    }
+
     // Case sensitive form of prefix query
     public final Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
         return prefixQuery(value, method, false, context);
@@ -433,6 +448,15 @@ protected final void failIfNotIndexed() {
         }
     }
 
+    protected final void failIfNotIndexedAndNoDocValues() {
+        // we fail if a field is both not indexed and does not have doc_values enabled
+        if (isIndexed == false && hasDocValues() == false) {
+            throw new IllegalArgumentException(
+                "Cannot search on field [" + name() + "] since it is both not indexed," + " and does not have doc_values enabled."
+            );
+        }
+    }
+
     public boolean eagerGlobalOrdinals() {
         return eagerGlobalOrdinals;
     }
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/CustomQueryScorer.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/CustomQueryScorer.java
index d0fb0f6da53c4..89c77b3cd403f 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/CustomQueryScorer.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/CustomQueryScorer.java
@@ -33,6 +33,7 @@
 package org.opensearch.search.fetch.subphase.highlight;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.WeightedSpanTerm;
@@ -104,6 +105,8 @@ protected void extract(Query query, float boost, Map t
                 super.extract(((FunctionScoreQuery) query).getSubQuery(), boost, terms);
             } else if (query instanceof OpenSearchToParentBlockJoinQuery) {
                 super.extract(((OpenSearchToParentBlockJoinQuery) query).getChildQuery(), boost, terms);
+            } else if (query instanceof IndexOrDocValuesQuery) {
+                super.extract(((IndexOrDocValuesQuery) query).getIndexQuery(), boost, terms);
             } else {
                 super.extract(query, boost, terms);
             }
diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java
index 0d8ef6784a28c..393c448330142 100644
--- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java
@@ -44,13 +44,17 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.NormsFieldExistsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.Operations;
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
@@ -128,14 +132,29 @@ public void testTermsQuery() {
         List terms = new ArrayList<>();
         terms.add(new BytesRef("foo"));
         terms.add(new BytesRef("bar"));
-        assertEquals(new TermInSetQuery("field", terms), ft.termsQuery(Arrays.asList("foo", "bar"), null));
+        Query expected = new IndexOrDocValuesQuery(
+            new TermInSetQuery("field", terms),
+            new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms)
+        );
+        assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null));
 
-        MappedFieldType unsearchable = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());
+        Query expectedIndex = new TermInSetQuery("field", terms);
+        assertEquals(expectedIndex, onlyIndexed.termsQuery(Arrays.asList("foo", "bar"), null));
+
+        MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        Query expectedDocValues = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms);
+        assertEquals(expectedDocValues, onlyDocValues.termsQuery(Arrays.asList("foo", "bar"), null));
+
+        MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap());
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> unsearchable.termsQuery(Arrays.asList("foo", "bar"), null)
         );
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals(
+            "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.",
+            e.getMessage()
+        );
     }
 
     public void testExistsQuery() {
@@ -157,9 +176,36 @@ public void testExistsQuery() {
 
     public void testRangeQuery() {
         MappedFieldType ft = new KeywordFieldType("field");
+
+        Query indexExpected = new TermRangeQuery("field", BytesRefs.toBytesRef("foo"), BytesRefs.toBytesRef("bar"), true, false);
+        Query dvExpected = new TermRangeQuery(
+            "field",
+            BytesRefs.toBytesRef("foo"),
+            BytesRefs.toBytesRef("bar"),
+            true,
+            false,
+            MultiTermQuery.DOC_VALUES_REWRITE
+        );
+
+        Query expected = new IndexOrDocValuesQuery(indexExpected, dvExpected);
+        Query actual = ft.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC);
+        assertEquals(expected, actual);
+
+        MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());
+        assertEquals(indexExpected, onlyIndexed.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC));
+
+        MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        assertEquals(dvExpected, onlyDocValues.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC));
+
+        MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap());
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> unsearchable.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC)
+        );
+
         assertEquals(
-            new TermRangeQuery("field", BytesRefs.toBytesRef("foo"), BytesRefs.toBytesRef("bar"), true, false),
-            ft.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC)
+            "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.",
+            e.getMessage()
         );
 
         OpenSearchException ee = expectThrows(
@@ -175,16 +221,37 @@ public void testRangeQuery() {
     public void testRegexpQuery() {
         MappedFieldType ft = new KeywordFieldType("field");
         assertEquals(
-            new RegexpQuery(new Term("field", "foo.*")),
+            new IndexOrDocValuesQuery(
+                new RegexpQuery(new Term("field", "foo.*")),
+                new RegexpQuery(new Term("field", "foo.*"), 0, 0, RegexpQuery.DEFAULT_PROVIDER, 10, MultiTermQuery.DOC_VALUES_REWRITE)
+            ),
             ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)
         );
 
-        MappedFieldType unsearchable = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        Query indexExpected = new RegexpQuery(new Term("field", "foo.*"));
+        MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());
+        assertEquals(indexExpected, onlyIndexed.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC));
+
+        Query dvExpected = new RegexpQuery(
+            new Term("field", "foo.*"),
+            0,
+            0,
+            RegexpQuery.DEFAULT_PROVIDER,
+            10,
+            MultiTermQuery.DOC_VALUES_REWRITE
+        );
+        MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        assertEquals(dvExpected, onlyDocValues.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC));
+
+        MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap());
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> unsearchable.regexpQuery("foo.*", 0, 0, 10, null, MOCK_QSC)
         );
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals(
+            "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.",
+            e.getMessage()
+        );
 
         OpenSearchException ee = expectThrows(
             OpenSearchException.class,
@@ -200,12 +267,26 @@ public void testFuzzyQuery() {
             ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MOCK_QSC)
         );
 
-        MappedFieldType unsearchable = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        Query indexExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true);
+        MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());
+        assertEquals(indexExpected, onlyIndexed.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MOCK_QSC));
+
+        Query dvExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE);
+        MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        assertEquals(
+            dvExpected,
+            onlyDocValues.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC)
+        );
+
+        MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap());
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> unsearchable.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MOCK_QSC)
+            () -> unsearchable.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC)
+        );
+        assertEquals(
+            "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.",
+            e.getMessage()
         );
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
 
         OpenSearchException ee = expectThrows(
             OpenSearchException.class,
@@ -214,6 +295,47 @@ public void testFuzzyQuery() {
         assertEquals("[fuzzy] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage());
     }
 
+    public void testWildCardQuery() {
+        MappedFieldType ft = new KeywordFieldType("field");
+        Query expected = new IndexOrDocValuesQuery(
+            new WildcardQuery(new Term("field", new BytesRef("foo*"))),
+            new WildcardQuery(
+                new Term("field", new BytesRef("foo*")),
+                Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
+                MultiTermQuery.DOC_VALUES_REWRITE
+            )
+        );
+        assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC));
+
+        Query indexExpected = new WildcardQuery(new Term("field", new BytesRef("foo*")));
+        MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());
+        assertEquals(indexExpected, onlyIndexed.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC));
+
+        Query dvExpected = new WildcardQuery(
+            new Term("field", new BytesRef("foo*")),
+            Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
+            MultiTermQuery.DOC_VALUES_REWRITE
+        );
+        MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap());
+        assertEquals(dvExpected, onlyDocValues.wildcardQuery("foo*", MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC));
+
+        MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap());
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> unsearchable.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)
+        );
+        assertEquals(
+            "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.",
+            e.getMessage()
+        );
+
+        OpenSearchException ee = expectThrows(
+            OpenSearchException.class,
+            () -> ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_DISALLOW_EXPENSIVE)
+        );
+        assertEquals("[wildcard] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage());
+    }
+
     public void testNormalizeQueries() {
         MappedFieldType ft = new KeywordFieldType("field");
         assertEquals(new TermQuery(new Term("field", new BytesRef("FOO"))), ft.termQuery("FOO", null));
diff --git a/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java
index e1391393f44fa..39f5bb313fe9e 100644
--- a/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java
@@ -304,10 +304,16 @@ public void testToQueryBooleanPrefixMultipleFields() throws IOException {
                 } else if (disjunct instanceof PrefixQuery) {
                     final PrefixQuery secondDisjunct = (PrefixQuery) disjunct;
                     assertThat(secondDisjunct.getPrefix(), equalTo(new Term(KEYWORD_FIELD_NAME, "foo bar")));
+                } else if (disjunct instanceof IndexOrDocValuesQuery) {
+                    final IndexOrDocValuesQuery iodvqDisjunct = (IndexOrDocValuesQuery) disjunct;
+                    assertThat(iodvqDisjunct.getIndexQuery().toString(), equalTo("mapped_string_2:foo bar*"));
                 } else {
                     throw new AssertionError();
                 }
-                assertThat(disjunct, either(instanceOf(BooleanQuery.class)).or(instanceOf(PrefixQuery.class)));
+                assertThat(
+                    disjunct,
+                    either(instanceOf(BooleanQuery.class)).or(instanceOf(PrefixQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class))
+                );
             }
         }
     }

From a2febe956defae25417a84bca3646efc2bafb7a5 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Tue, 31 Oct 2023 01:21:42 -0700
Subject: [PATCH 261/388] Fix SegRep bug where primary shards do not update
 tracked replica state post failover/relocation (#11017)

Signed-off-by: Marc Handalian 
---
 .../replication/SegmentReplicationIT.java     | 57 +++++++++++++++++++
 .../SegmentReplicationTargetService.java      |  7 +++
 .../SegmentReplicationTargetServiceTests.java |  3 +-
 3 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
index f48df082a25dc..a2996d87a851b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.index.StandardDirectoryReader;
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.flush.FlushRequest;
 import org.opensearch.action.admin.indices.stats.IndicesStatsRequest;
@@ -62,6 +63,7 @@
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.IndexModule;
+import org.opensearch.index.ReplicationStats;
 import org.opensearch.index.SegmentReplicationPerGroupStats;
 import org.opensearch.index.SegmentReplicationPressureService;
 import org.opensearch.index.SegmentReplicationShardStats;
@@ -94,6 +96,7 @@
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import static java.util.Arrays.asList;
@@ -1776,4 +1779,58 @@ public void testRealtimeTermVectorRequestsUnSuccessful() throws IOException {
         assertThat(response.getIndex(), equalTo(INDEX_NAME));
 
     }
+
+    public void testReplicaAlreadyAtCheckpoint() throws Exception {
+        final List nodes = new ArrayList<>();
+        final String primaryNode = internalCluster().startDataOnlyNode();
+        nodes.add(primaryNode);
+        final Settings settings = Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build();
+        createIndex(INDEX_NAME, settings);
+        ensureGreen(INDEX_NAME);
+        // start a replica node, initially will be empty with no shard assignment.
+        final String replicaNode = internalCluster().startDataOnlyNode();
+        nodes.add(replicaNode);
+        final String replicaNode2 = internalCluster().startDataOnlyNode();
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareUpdateSettings(INDEX_NAME)
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2))
+        );
+        ensureGreen(INDEX_NAME);
+
+        // index a doc.
+        client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", randomInt()).get();
+        refresh(INDEX_NAME);
+
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
+        ensureYellowAndNoInitializingShards(INDEX_NAME);
+        IndexShard replica_1 = getIndexShard(replicaNode, INDEX_NAME);
+        IndexShard replica_2 = getIndexShard(replicaNode2, INDEX_NAME);
+        // wait until a replica is promoted & finishes engine flip, we don't care which one
+        AtomicReference primary = new AtomicReference<>();
+        assertBusy(() -> {
+            assertTrue("replica should be promoted as a primary", replica_1.routingEntry().primary() || replica_2.routingEntry().primary());
+            primary.set(replica_1.routingEntry().primary() ? replica_1 : replica_2);
+        });
+
+        FlushRequest request = new FlushRequest(INDEX_NAME);
+        request.force(true);
+        primary.get().flush(request);
+
+        assertBusy(() -> {
+            assertEquals(
+                replica_1.getLatestReplicationCheckpoint().getSegmentInfosVersion(),
+                replica_2.getLatestReplicationCheckpoint().getSegmentInfosVersion()
+            );
+        });
+
+        assertBusy(() -> {
+            ClusterStatsResponse clusterStatsResponse = client().admin().cluster().prepareClusterStats().get();
+            ReplicationStats replicationStats = clusterStatsResponse.getIndicesStats().getSegments().getReplicationStats();
+            assertEquals(0L, replicationStats.maxBytesBehind);
+            assertEquals(0L, replicationStats.maxReplicationLag);
+            assertEquals(0L, replicationStats.totalBytesBehind);
+        });
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
index 46095adfe96b4..73da0482537ad 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
@@ -53,6 +53,7 @@
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicLong;
 
+import static org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
 import static org.opensearch.indices.replication.SegmentReplicationSourceService.Actions.UPDATE_VISIBLE_CHECKPOINT;
 
 /**
@@ -282,6 +283,12 @@ public void onReplicationFailure(
                         }
                     }
                 });
+            } else if (replicaShard.isSegmentReplicationAllowed()) {
+                // if we didn't process the checkpoint because we are up to date,
+                // send our latest checkpoint to the primary to update tracking.
+                // replicationId is not used by the primary set to a default value.
+                final long replicationId = NO_OPS_PERFORMED;
+                updateVisibleCheckpoint(replicationId, replicaShard);
             }
         } else {
             logger.trace(
diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
index 7b02635525264..252f3975bab25 100644
--- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
@@ -55,7 +55,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 
-import static org.junit.Assert.assertEquals;
+import static org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.atLeastOnce;
@@ -247,6 +247,7 @@ public void testAlreadyOnNewCheckpoint() {
         SegmentReplicationTargetService spy = spy(sut);
         spy.onNewCheckpoint(replicaShard.getLatestReplicationCheckpoint(), replicaShard);
         verify(spy, times(0)).startReplication(any(), any(), any());
+        verify(spy, times(1)).updateVisibleCheckpoint(NO_OPS_PERFORMED, replicaShard);
     }
 
     @TestLogging(reason = "Getting trace logs from replication package", value = "org.opensearch.indices.replication:TRACE")

From cb39d0057e7277e02a99e2cf5dabfc36855991c4 Mon Sep 17 00:00:00 2001
From: Anshu Agarwal 
Date: Tue, 31 Oct 2023 20:07:17 +0530
Subject: [PATCH 262/388] Fix weighted routing flaky test (#10955)

Signed-off-by: Anshu Agarwal 
Co-authored-by: Anshu Agarwal 
---
 .../opensearch/search/SearchWeightedRoutingIT.java    | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java
index 5207dab83f1d9..aa1fe695ecc12 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java
@@ -501,8 +501,9 @@ public void testShardRoutingWithNetworkDisruption_FailOpenEnabled() throws Excep
 
         logger.info("--> creating network partition disruption");
         final String clusterManagerNode1 = internalCluster().getClusterManagerName();
-        Set nodesInOneSide = Stream.of(clusterManagerNode1, nodeMap.get("b").get(0)).collect(Collectors.toCollection(HashSet::new));
-        Set nodesInOtherSide = Stream.of(nodeMap.get("a").get(0)).collect(Collectors.toCollection(HashSet::new));
+        Set nodesInOneSide = Stream.of(nodeMap.get("a").get(0)).collect(Collectors.toCollection(HashSet::new));
+        Set nodesInOtherSide = Stream.of(clusterManagerNode1, nodeMap.get("b").get(0), nodeMap.get("c").get(0))
+            .collect(Collectors.toCollection(HashSet::new));
 
         NetworkDisruption networkDisruption = new NetworkDisruption(
             new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInOtherSide),
@@ -870,8 +871,7 @@ private void assertSearchInAZ(String az) {
             SearchStats.Stats searchStats = stat.getIndices().getSearch().getTotal();
             if (stat.getNode().isDataNode()) {
                 if (stat.getNode().getId().equals(dataNodeId)) {
-                    Assert.assertTrue(searchStats.getFetchCount() > 0L);
-                    Assert.assertTrue(searchStats.getQueryCount() > 0L);
+                    Assert.assertTrue(searchStats.getFetchCount() > 0L || searchStats.getQueryCount() > 0L);
                 }
             }
         }
@@ -945,7 +945,6 @@ public void testSearchAggregationWithNetworkDisruption_FailOpenEnabled() throws
         }
 
         logger.info("--> network disruption is stopped");
-        networkDisruption.stopDisrupting();
 
         for (int i = 0; i < 50; i++) {
             try {
@@ -962,6 +961,8 @@ public void testSearchAggregationWithNetworkDisruption_FailOpenEnabled() throws
                 fail("search should not fail");
             }
         }
+        networkDisruption.stopDisrupting();
+
         assertSearchInAZ("b");
         assertSearchInAZ("c");
         assertNoSearchInAZ("a");

From aa0fddb50d6150330718ac0af23b72bd1598ae4a Mon Sep 17 00:00:00 2001
From: Harsha Vamsi Kalluri 
Date: Tue, 31 Oct 2023 17:36:35 -0700
Subject: [PATCH 263/388] Fix changelog for multiterm perf PR (#11038)

Signed-off-by: Harsha Vamsi Kalluri 
---
 CHANGELOG.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 046693421d07b..54438d28ad35f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -63,7 +63,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855))
 - Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
 - Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558))
-- Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 
 
 ### Deprecated
@@ -125,6 +124,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
 - [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
+- Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 
 ### Deprecated
 
@@ -141,4 +141,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x

From a512dae4ac84792018cc81d5eacc8c69deefda1a Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Wed, 1 Nov 2023 07:58:00 -0400
Subject: [PATCH 264/388] Add Java 11/17/21 matrix for precommit and assemble
 checks (#11035)

Signed-off-by: Andriy Redko 
---
 .github/workflows/assemble.yml  | 5 +++--
 .github/workflows/precommit.yml | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
index 6a66ac5fb5609..87cecdf38c072 100644
--- a/.github/workflows/assemble.yml
+++ b/.github/workflows/assemble.yml
@@ -7,13 +7,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        java: [ 11, 17, 21 ]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 11
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
           distribution: temurin
       - name: Setup docker (missing on MacOS)
         if: runner.os == 'macos'
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
index b04f404b11c55..cd75eb47946a4 100644
--- a/.github/workflows/precommit.yml
+++ b/.github/workflows/precommit.yml
@@ -7,13 +7,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        java: [ 11, 17, 21 ]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 11
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
       - name: Run Gradle (precommit)

From 8673fa937db405b8d614f8d4a02c0aa52587c037 Mon Sep 17 00:00:00 2001
From: Bharathwaj G 
Date: Wed, 1 Nov 2023 21:12:35 +0530
Subject: [PATCH 265/388] Fix testDeleteAllAndListAllPits and
 testDeleteWhileSearch flaky tests (#10946)

Signed-off-by: Bharathwaj G 
---
 .../java/org/opensearch/client/PitIT.java     | 21 +++++++++----------
 .../search/pit/DeletePitMultiNodeIT.java      |  6 +++++-
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/PitIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/PitIT.java
index d5c1888e78b5d..b0990560b08ba 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/PitIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/PitIT.java
@@ -24,7 +24,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -72,7 +71,7 @@ public void testCreateAndDeletePit() throws IOException {
         assertTrue(deletePitResponse.getDeletePitResults().get(0).getPitId().equals(createPitResponse.getId()));
     }
 
-    public void testDeleteAllAndListAllPits() throws IOException, InterruptedException {
+    public void testDeleteAllAndListAllPits() throws Exception {
         CreatePitRequest pitRequest = new CreatePitRequest(new TimeValue(1, TimeUnit.DAYS), true, "index");
         CreatePitResponse pitResponse = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
         CreatePitResponse pitResponse1 = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
@@ -91,11 +90,9 @@ public void testDeleteAllAndListAllPits() throws IOException, InterruptedExcepti
         List pits = getAllPitResponse.getPitInfos().stream().map(r -> r.getPitId()).collect(Collectors.toList());
         assertTrue(pits.contains(pitResponse.getId()));
         assertTrue(pits.contains(pitResponse1.getId()));
-        CountDownLatch countDownLatch = new CountDownLatch(1);
         ActionListener deletePitListener = new ActionListener<>() {
             @Override
             public void onResponse(DeletePitResponse response) {
-                countDownLatch.countDown();
                 for (DeletePitInfo deletePitInfo : response.getDeletePitResults()) {
                     assertTrue(deletePitInfo.isSuccessful());
                 }
@@ -103,19 +100,20 @@ public void onResponse(DeletePitResponse response) {
 
             @Override
             public void onFailure(Exception e) {
-                countDownLatch.countDown();
                 if (!(e instanceof OpenSearchStatusException)) {
                     throw new AssertionError("Delete all failed");
                 }
             }
         };
         final CreatePitResponse pitResponse3 = execute(pitRequest, highLevelClient()::createPit, highLevelClient()::createPitAsync);
-
+        assertTrue(pitResponse3.getId() != null);
         ActionListener getPitsListener = new ActionListener() {
             @Override
             public void onResponse(GetAllPitNodesResponse response) {
                 List pits = response.getPitInfos().stream().map(r -> r.getPitId()).collect(Collectors.toList());
                 assertTrue(pits.contains(pitResponse3.getId()));
+                // delete all pits
+                highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
             }
 
             @Override
@@ -126,11 +124,12 @@ public void onFailure(Exception e) {
             }
         };
         highLevelClient().getAllPitsAsync(RequestOptions.DEFAULT, getPitsListener);
-        highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
-        assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
+
         // validate no pits case
-        getAllPitResponse = highLevelClient().getAllPits(RequestOptions.DEFAULT);
-        assertTrue(getAllPitResponse.getPitInfos().size() == 0);
-        highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
+        assertBusy(() -> {
+            GetAllPitNodesResponse getAllPitResponse1 = highLevelClient().getAllPits(RequestOptions.DEFAULT);
+            assertTrue(getAllPitResponse1.getPitInfos().size() == 0);
+            highLevelClient().deleteAllPitsAsync(RequestOptions.DEFAULT, deletePitListener);
+        });
     }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java
index 4a178e7066846..8ae652082f653 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/pit/DeletePitMultiNodeIT.java
@@ -309,7 +309,11 @@ public void testDeleteWhileSearch() throws Exception {
     private void verifySearchContextMissingException(ShardSearchFailure[] failures) {
         for (ShardSearchFailure failure : failures) {
             Throwable cause = ExceptionsHelper.unwrapCause(failure.getCause());
-            assertTrue(failure.toString(), cause instanceof SearchContextMissingException);
+            if (failure.toString().contains("reader_context is already closed can't increment refCount current count")) {
+                // this is fine, expected search error when context is already deleted
+            } else {
+                assertTrue(failure.toString(), cause instanceof SearchContextMissingException);
+            }
         }
     }
 

From c851b34735294a4eb6baf5c1254de1b130daa6c4 Mon Sep 17 00:00:00 2001
From: Henri Yandell <477715+hyandell@users.noreply.github.com>
Date: Thu, 2 Nov 2023 16:38:30 -0700
Subject: [PATCH 266/388] Adding slf4j license header to
 LoggerMessageFormat.java (#11069)

* Adding slf4j license header per #9879

Signed-off-by: Henri Yandell 
Signed-off-by: Henri Yandell <477715+hyandell@users.noreply.github.com>
Signed-off-by: Peter Nied 
Co-authored-by: Peter Nied 
---
 CHANGELOG.md                                               | 1 +
 .../core/common/logging/LoggerMessageFormat.java           | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 54438d28ad35f..9eb3486701641 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -100,6 +100,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
 - Added cluster setting cluster.restrict.index.replication_type to restrict setting of index setting replication type ([#10866](https://github.com/opensearch-project/OpenSearch/pull/10866))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
+- Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java b/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java
index cd75bddd680e5..c7b9bee3cbf4d 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/logging/LoggerMessageFormat.java
@@ -30,6 +30,13 @@
  * GitHub history for details.
  */
 
+/*
+ * This code is based on code from SFL4J 1.5.11
+ * Copyright (c) 2004-2007 QOS.ch
+ * All rights reserved.
+ * SPDX-License-Identifier: MIT
+ */
+
 package org.opensearch.core.common.logging;
 
 import java.util.HashSet;

From 38999b2f8c796d1575275b09644a9e99f5145fbb Mon Sep 17 00:00:00 2001
From: Poojita Raj 
Date: Thu, 2 Nov 2023 16:53:29 -0700
Subject: [PATCH 267/388] =?UTF-8?q?Revert=20"Add=20cluster=20setting=20clu?=
 =?UTF-8?q?ster.restrict.index.replication=5Ftype=20t=E2=80=A6=20(#10866)"?=
 =?UTF-8?q?=20(#11072)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 8b2173910f754a48773b3283e1a511cbc1a9db78.

Signed-off-by: Poojita Raj 
---
 CHANGELOG.md                                  |  1 -
 .../SegmentReplicationClusterSettingIT.java   | 27 -------------------
 .../metadata/MetadataCreateIndexService.java  | 19 -------------
 .../common/settings/ClusterSettings.java      |  3 +--
 .../opensearch/indices/IndicesService.java    | 11 --------
 .../MetadataCreateIndexServiceTests.java      | 16 ++---------
 6 files changed, 3 insertions(+), 74 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9eb3486701641..2753249c9d956 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -98,7 +98,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce ConcurrentQueryProfiler to profile query using concurrent segment search path and support concurrency during rewrite and create weight ([10352](https://github.com/opensearch-project/OpenSearch/pull/10352))
 - Update the indexRandom function to create more segments for concurrent search tests ([10247](https://github.com/opensearch-project/OpenSearch/pull/10247))
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
-- Added cluster setting cluster.restrict.index.replication_type to restrict setting of index setting replication type ([#10866](https://github.com/opensearch-project/OpenSearch/pull/10866))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
 - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java
index 186a5ce39f131..a82fd8d845709 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java
@@ -19,7 +19,6 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
-import static org.opensearch.indices.IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
@@ -124,30 +123,4 @@ public void testIndexReplicationSettingOverridesDocRepClusterSetting() throws Ex
         assertEquals(indicesService.indexService(anotherIndex).getIndexSettings().isSegRepEnabled(), false);
     }
 
-    public void testIndexReplicationTypeWhenRestrictSettingTrue() {
-        testRestrictIndexReplicationTypeSetting(true, randomFrom(ReplicationType.values()));
-    }
-
-    public void testIndexReplicationTypeWhenRestrictSettingFalse() {
-        testRestrictIndexReplicationTypeSetting(false, randomFrom(ReplicationType.values()));
-    }
-
-    private void testRestrictIndexReplicationTypeSetting(boolean setRestrict, ReplicationType replicationType) {
-        String expectedExceptionMsg =
-            "Validation Failed: 1: index setting [index.replication.type] is not allowed to be set as [cluster.restrict.index.replication_type=true];";
-        String clusterManagerName = internalCluster().startNode(
-            Settings.builder().put(CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey(), setRestrict).build()
-        );
-        internalCluster().startDataOnlyNodes(1);
-
-        // Test create index fails
-        Settings indexSettings = Settings.builder().put(indexSettings()).put(SETTING_REPLICATION_TYPE, replicationType).build();
-        if (setRestrict) {
-            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME, indexSettings));
-            assertEquals(expectedExceptionMsg, exception.getMessage());
-        } else {
-            createIndex(INDEX_NAME, indexSettings);
-        }
-    }
-
 }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 78a22fe11f072..8d76a39712ee3 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -1252,7 +1252,6 @@ List getIndexSettingsValidationErrors(
         if (forbidPrivateIndexSettings) {
             validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings));
         }
-        validateIndexReplicationTypeSettings(settings, clusterService.getClusterSettings()).ifPresent(validationErrors::add);
         if (indexName.isEmpty() || indexName.get().charAt(0) != '.') {
             // Apply aware replica balance validation only to non system indices
             int replicaCount = settings.getAsInt(
@@ -1307,24 +1306,6 @@ private static List validateIndexCustomPath(Settings settings, @Nullable
         return validationErrors;
     }
 
-    /**
-     * Validates {@code index.replication.type} is not set if {@code cluster.restrict.index.replication_type} is set to true.
-     *
-     * @param requestSettings settings passed in during index create request
-     * @param clusterSettings cluster setting
-     */
-    private static Optional validateIndexReplicationTypeSettings(Settings requestSettings, ClusterSettings clusterSettings) {
-        if (requestSettings.hasValue(SETTING_REPLICATION_TYPE)
-            && clusterSettings.get(IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING)) {
-            return Optional.of(
-                "index setting [index.replication.type] is not allowed to be set as ["
-                    + IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey()
-                    + "=true]"
-            );
-        }
-        return Optional.empty();
-    }
-
     /**
      * Validates the settings and mappings for shrinking an index.
      *
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 3a1fff21db366..5ab1f49949679 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -691,8 +691,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
                 CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
-                CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
-                IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING
+                CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT
             )
         )
     );
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index 36abc77893d81..50c551c2be29b 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -299,17 +299,6 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Final
     );
 
-    /**
-     * This setting is used to restrict creation of index where the 'index.replication.type' index setting is set.
-     * If disabled, the replication type can be specified.
-     */
-    public static final Setting CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING = Setting.boolSetting(
-        "cluster.restrict.index.replication_type",
-        false,
-        Property.NodeScope,
-        Property.Final
-    );
-
     /**
      * The node's settings.
      */
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index cace66d8c6d9e..e40826915c848 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -139,7 +139,6 @@
 import static org.opensearch.indices.IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
-import static org.opensearch.indices.IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING;
 import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService;
 import static org.opensearch.node.Node.NODE_ATTRIBUTES;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
@@ -1178,8 +1177,6 @@ public void testvalidateIndexSettings() {
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b")
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e")
             .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true)
-            .put(CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey(), true)
-            .put(SETTING_REPLICATION_TYPE, randomFrom(ReplicationType.values()))
             .build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         when(clusterService.getSettings()).thenReturn(settings);
@@ -1203,12 +1200,8 @@ public void testvalidateIndexSettings() {
         );
 
         List validationErrors = checkerService.getIndexSettingsValidationErrors(settings, false, Optional.empty());
-        assertThat(validationErrors.size(), is(2));
-        assertThat(
-            validationErrors.get(0),
-            is("index setting [index.replication.type] is not allowed to be set as [cluster.restrict.index.replication_type=true]")
-        );
-        assertThat(validationErrors.get(1), is("expected total copies needs to be a multiple of total awareness attributes [3]"));
+        assertThat(validationErrors.size(), is(1));
+        assertThat(validationErrors.get(0), is("expected total copies needs to be a multiple of total awareness attributes [3]"));
 
         settings = Settings.builder()
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack")
@@ -1216,13 +1209,8 @@ public void testvalidateIndexSettings() {
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e")
             .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true)
             .put(SETTING_NUMBER_OF_REPLICAS, 2)
-            .put(CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING.getKey(), false)
-            .put(SETTING_REPLICATION_TYPE, randomFrom(ReplicationType.values()))
             .build();
 
-        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
-
         validationErrors = checkerService.getIndexSettingsValidationErrors(settings, false, Optional.empty());
         assertThat(validationErrors.size(), is(0));
 

From 54fa0508130280c8f64c9f674b8104844d86af91 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Thu, 2 Nov 2023 18:55:59 -0700
Subject: [PATCH 268/388] Refactor SegmentReplicationTargetService to only hold
 completed target state instead of the entire target. (#11043)

Signed-off-by: Marc Handalian 
---
 .../replication/SegmentReplicationTargetService.java        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
index 73da0482537ad..cb738d74000bc 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
@@ -70,7 +70,7 @@ public class SegmentReplicationTargetService implements IndexEventListener {
 
     private final ReplicationCollection onGoingReplications;
 
-    private final Map completedReplications = ConcurrentCollections.newConcurrentMap();
+    private final Map completedReplications = ConcurrentCollections.newConcurrentMap();
 
     private final SegmentReplicationSourceFactory sourceFactory;
 
@@ -192,7 +192,7 @@ public SegmentReplicationState getOngoingEventSegmentReplicationState(ShardId sh
      */
     @Nullable
     public SegmentReplicationState getlatestCompletedEventSegmentReplicationState(ShardId shardId) {
-        return Optional.ofNullable(completedReplications.get(shardId)).map(SegmentReplicationTarget::state).orElse(null);
+        return completedReplications.get(shardId);
     }
 
     /**
@@ -525,7 +525,7 @@ public void onResponse(Void o) {
                 logger.debug(() -> new ParameterizedMessage("Finished replicating {} marking as done.", target.description()));
                 onGoingReplications.markAsDone(replicationId);
                 if (target.state().getIndex().recoveredFileCount() != 0 && target.state().getIndex().recoveredBytes() != 0) {
-                    completedReplications.put(target.shardId(), target);
+                    completedReplications.put(target.shardId(), target.state());
                 }
             }
 

From 1130d656a64a3c8e7476c5776b9e6c9efe937255 Mon Sep 17 00:00:00 2001
From: Jay Deng 
Date: Fri, 3 Nov 2023 18:47:30 -0700
Subject: [PATCH 269/388] Disable concurrent aggs for Diversified Sampler and
 Sampler aggs (#11087)

Signed-off-by: Jay Deng 
---
 CHANGELOG.md                                  |  1 +
 .../bucket/DiversifiedSamplerIT.java          |  9 +++-----
 .../search/aggregations/bucket/SamplerIT.java | 22 +++++++++++++++++--
 .../sampler/DiversifiedAggregatorFactory.java |  2 +-
 .../sampler/SamplerAggregatorFactory.java     |  2 +-
 5 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2753249c9d956..0b8f1084eafd4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -125,6 +125,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
 - [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
 - Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
+- Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
 
 ### Deprecated
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java
index 865dd670fbf68..1d5f7f93e7410 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DiversifiedSamplerIT.java
@@ -33,9 +33,9 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.opensearch.action.admin.indices.refresh.RefreshRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchType;
+import org.opensearch.action.support.WriteRequest;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.query.TermQueryBuilder;
@@ -132,13 +132,14 @@ public void setupSuiteScopeCluster() throws Exception {
             client().prepareIndex("test")
                 .setId("" + i)
                 .setSource("author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3]))
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                 .get();
             client().prepareIndex("idx_unmapped_author")
                 .setId("" + i)
                 .setSource("name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3]))
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                 .get();
         }
-        client().admin().indices().refresh(new RefreshRequest("test")).get();
     }
 
     public void testIssue10719() throws Exception {
@@ -221,10 +222,6 @@ public void testNestedDiversity() throws Exception {
     }
 
     public void testNestedSamples() throws Exception {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10046",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         // Test samples nested under samples
         int MAX_DOCS_PER_AUTHOR = 1;
         int MAX_DOCS_PER_GENRE = 2;
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java
index 7033c42c5d661..c7b03d21cb6bb 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/SamplerIT.java
@@ -34,9 +34,9 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.opensearch.action.admin.indices.refresh.RefreshRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchType;
+import org.opensearch.action.support.WriteRequest;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.query.TermQueryBuilder;
@@ -132,13 +132,14 @@ public void setupSuiteScopeCluster() throws Exception {
             client().prepareIndex("test")
                 .setId("" + i)
                 .setSource("author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3]))
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                 .get();
             client().prepareIndex("idx_unmapped_author")
                 .setId("" + i)
                 .setSource("name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3]))
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                 .get();
         }
-        client().admin().indices().refresh(new RefreshRequest("test")).get();
     }
 
     public void testIssue10719() throws Exception {
@@ -195,6 +196,23 @@ public void testSimpleSampler() throws Exception {
         assertThat(maxBooksPerAuthor, equalTo(3L));
     }
 
+    public void testSimpleSamplerShardSize() throws Exception {
+        final int SHARD_SIZE = randomIntBetween(1, 3);
+        SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(SHARD_SIZE);
+        sampleAgg.subAggregation(terms("authors").field("author"));
+        SearchResponse response = client().prepareSearch("test")
+            .setSearchType(SearchType.QUERY_THEN_FETCH)
+            .setQuery(new TermQueryBuilder("genre", "fantasy"))
+            .setFrom(0)
+            .setSize(60)
+            .addAggregation(sampleAgg)
+            .get();
+        assertSearchResponse(response);
+        Sampler sample = response.getAggregations().get("sample");
+        Terms authors = sample.getAggregations().get("authors");
+        assertEquals(SHARD_SIZE * NUM_SHARDS, sample.getDocCount());
+    }
+
     public void testUnmappedChildAggNoDiversity() throws Exception {
         SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(100);
         sampleAgg.subAggregation(terms("authors").field("author"));
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
index 5f81c76b69385..0f3c9872353c1 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
@@ -162,6 +162,6 @@ public InternalAggregation buildEmptyAggregation() {
 
     @Override
     protected boolean supportsConcurrentSegmentSearch() {
-        return true;
+        return false;
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
index d3db8a66ee21f..51d9830d3cea0 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
@@ -75,6 +75,6 @@ public Aggregator createInternal(
 
     @Override
     protected boolean supportsConcurrentSegmentSearch() {
-        return true;
+        return false;
     }
 }

From 747f7d1550c5d9ffe9951902d89a15ed60cdf39f Mon Sep 17 00:00:00 2001
From: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com>
Date: Sat, 4 Nov 2023 20:47:20 -0700
Subject: [PATCH 270/388] Fixing concurrent search tests with one slice
 (#11071)

* Fixing concurrent search tests with one slice (#11071)

Signed-off-by: Ticheng Lin 

* Remove changes for non-flaky tests (#11071)

Signed-off-by: Ticheng Lin 

---------

Signed-off-by: Ticheng Lin 
---
 .../search/nested/SimpleNestedIT.java           |  8 ++++++++
 .../opensearch/search/pit/PitMultiNodeIT.java   |  1 +
 .../search/preference/SearchPreferenceIT.java   | 10 ++++++----
 .../opensearch/search/query/QueryStringIT.java  |  6 ++++++
 .../search/query/ScriptScoreQueryIT.java        | 12 ++++++++----
 .../search/query/SimpleQueryStringIT.java       | 17 ++++++++++++++++-
 6 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java
index 83dec7b27a897..656e7b2e366ed 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java
@@ -293,6 +293,7 @@ public void testMultiNested() throws Exception {
         refresh();
         // check the numDocs
         assertDocumentCount("test", 7);
+        indexRandomForConcurrentSearch("test");
 
         // do some multi nested queries
         SearchResponse searchResponse = client().prepareSearch("test")
@@ -485,6 +486,7 @@ public void testExplain() throws Exception {
             )
             .setRefreshPolicy(IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1"), ScoreMode.Total))
@@ -968,6 +970,10 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception {
 
     // https://github.com/elastic/elasticsearch/issues/31554
     public void testLeakingSortValues() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11065",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 1))
                 .setMapping(
@@ -1035,6 +1041,7 @@ public void testLeakingSortValues() throws Exception {
             .get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(termQuery("_id", 2))
@@ -1627,6 +1634,7 @@ public void testCheckFixedBitSetCache() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value").get();
         refresh();
         ensureSearchable("test");
+        indexRandomForConcurrentSearch("test");
 
         // No nested mapping yet, there shouldn't be anything in the fixed bit set cache
         ClusterStatsResponse clusterStatsResponse = client().admin().cluster().prepareClusterStats().get();
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java
index e42f12709c948..a3432bfe7e3e4 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java
@@ -100,6 +100,7 @@ public void clearIndex() {
     public void testPit() throws Exception {
         CreatePitRequest request = new CreatePitRequest(TimeValue.timeValueDays(1), true);
         request.setIndices(new String[] { "index" });
+        indexRandomForConcurrentSearch("index");
         ActionFuture execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse pitResponse = execute.get();
         SearchResponse searchResponse = client().prepareSearch("index")
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java
index 425764b1c88d2..97fe05f5b9747 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java
@@ -52,7 +52,6 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -99,7 +98,7 @@ public Settings nodeSettings(int nodeOrdinal) {
     }
 
     // see #2896
-    public void testStopOneNodePreferenceWithRedState() throws IOException {
+    public void testStopOneNodePreferenceWithRedState() throws Exception {
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder().put("index.number_of_shards", cluster().numDataNodes() + 2).put("index.number_of_replicas", 0)
@@ -110,6 +109,7 @@ public void testStopOneNodePreferenceWithRedState() throws IOException {
             client().prepareIndex("test").setId("" + i).setSource("field1", "value1").get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
         internalCluster().stopRandomDataNode();
         client().admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).get();
         String[] preferences = new String[] {
@@ -138,7 +138,7 @@ public void testStopOneNodePreferenceWithRedState() throws IOException {
         assertThat("_only_local", searchResponse.getFailedShards(), greaterThanOrEqualTo(0));
     }
 
-    public void testNoPreferenceRandom() {
+    public void testNoPreferenceRandom() throws Exception {
         assertAcked(
             prepareCreate("test").setSettings(
                 // this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
@@ -149,6 +149,7 @@ public void testNoPreferenceRandom() {
 
         client().prepareIndex("test").setSource("field1", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         final Client client = internalCluster().smartClient();
         SearchResponse searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()).get();
@@ -201,7 +202,7 @@ public void testThatSpecifyingNonExistingNodesReturnsUsefulError() {
         }
     }
 
-    public void testNodesOnlyRandom() {
+    public void testNodesOnlyRandom() throws Exception {
         assertAcked(
             prepareCreate("test").setSettings(
                 // this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
@@ -211,6 +212,7 @@ public void testNodesOnlyRandom() {
         ensureGreen();
         client().prepareIndex("test").setSource("field1", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         final Client client = internalCluster().smartClient();
         // multiple wildchar to cover multi-param usecase
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
index 099eb934f4f4d..1ca5859f23bca 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
@@ -186,6 +186,7 @@ public void testDocWithAllTypes() throws Exception {
         String docBody = copyToStringFromClasspath("/org/opensearch/search/query/all-example-document.json");
         reqs.add(client().prepareIndex("test").setId("1").setSource(docBody, MediaTypeRegistry.JSON));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
         assertHits(resp.getHits(), "1");
@@ -225,6 +226,7 @@ public void testKeywordWithWhitespace() throws Exception {
         reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar"));
         reqs.add(client().prepareIndex("test").setId("3").setSource("f1", "foo bar"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
         assertHits(resp.getHits(), "3");
@@ -245,6 +247,7 @@ public void testRegexCaseInsensitivity() throws Exception {
         indexRequests.add(client().prepareIndex("messages").setId("1").setSource("message", "message: this is a TLS handshake"));
         indexRequests.add(client().prepareIndex("messages").setId("2").setSource("message", "message: this is a tcp handshake"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("messages");
 
         SearchResponse response = client().prepareSearch("messages").setQuery(queryStringQuery("/TLS/").defaultField("message")).get();
         assertNoFailures(response);
@@ -282,6 +285,7 @@ public void testAllFields() throws Exception {
         List reqs = new ArrayList<>();
         reqs.add(client().prepareIndex("test_1").setId("1").setSource("f1", "foo", "f2", "eggplant"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test_1");
 
         SearchResponse resp = client().prepareSearch("test_1")
             .setQuery(queryStringQuery("foo eggplant").defaultOperator(Operator.AND))
@@ -374,6 +378,7 @@ public void testLimitOnExpandedFields() throws Exception {
 
         client().prepareIndex("testindex").setId("1").setSource("field_A0", "foo bar baz").get();
         refresh();
+        indexRandomForConcurrentSearch("testindex");
 
         // single field shouldn't trigger the limit
         doAssertOneHitForQueryString("field_A0:foo");
@@ -465,6 +470,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
         List indexRequests = new ArrayList<>();
         indexRequests.add(client().prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         // The wildcard field matches aliases for both a text and geo_point field.
         // By default, the geo_point field should be ignored when building the query.
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java
index 7ba582811bbc2..55029712a061c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java
@@ -109,13 +109,14 @@ protected Map, Object>> pluginScripts() {
     // 1) only matched docs retrieved
     // 2) score is calculated based on a script with params
     // 3) min score applied
-    public void testScriptScore() {
+    public void testScriptScore() throws Exception {
         assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
         int docCount = 10;
         for (int i = 1; i <= docCount; i++) {
             client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test-index");
 
         Map params = new HashMap<>();
         params.put("param1", 0.1);
@@ -135,13 +136,14 @@ public void testScriptScore() {
         assertOrderedSearchHits(resp, "10", "8", "6");
     }
 
-    public void testScriptScoreBoolQuery() {
+    public void testScriptScoreBoolQuery() throws Exception {
         assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
         int docCount = 10;
         for (int i = 1; i <= docCount; i++) {
             client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + i, "field2", i).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test-index");
 
         Map params = new HashMap<>();
         params.put("param1", 0.1);
@@ -155,7 +157,7 @@ public void testScriptScoreBoolQuery() {
     }
 
     // test that when the internal query is rewritten script_score works well
-    public void testRewrittenQuery() {
+    public void testRewrittenQuery() throws Exception {
         assertAcked(
             prepareCreate("test-index2").setSettings(Settings.builder().put("index.number_of_shards", 1))
                 .setMapping("field1", "type=date", "field2", "type=double")
@@ -164,6 +166,7 @@ public void testRewrittenQuery() {
         client().prepareIndex("test-index2").setId("2").setSource("field1", "2019-10-01", "field2", 2).get();
         client().prepareIndex("test-index2").setId("3").setSource("field1", "2019-11-01", "field2", 3).get();
         refresh();
+        indexRandomForConcurrentSearch("test-index2");
 
         RangeQueryBuilder rangeQB = new RangeQueryBuilder("field1").from("2019-01-01"); // the query should be rewritten to from:null
         Map params = new HashMap<>();
@@ -174,7 +177,7 @@ public void testRewrittenQuery() {
         assertOrderedSearchHits(resp, "3", "2", "1");
     }
 
-    public void testDisallowExpensiveQueries() {
+    public void testDisallowExpensiveQueries() throws Exception {
         try {
             assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
             int docCount = 10;
@@ -182,6 +185,7 @@ public void testDisallowExpensiveQueries() {
                 client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get();
             }
             refresh();
+            indexRandomForConcurrentSearch("test-index");
 
             Map params = new HashMap<>();
             params.put("param1", 0.1);
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
index 384d2b7423e66..017d28ef3a2a6 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
@@ -150,6 +150,7 @@ public void testSimpleQueryString() throws ExecutionException, InterruptedExcept
             client().prepareIndex("test").setId("5").setSource("body", "quux baz spaghetti"),
             client().prepareIndex("test").setId("6").setSource("otherbody", "spaghetti")
         );
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar")).get();
         assertHitCount(searchResponse, 3L);
@@ -199,6 +200,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
             client().prepareIndex("test").setId("3").setSource("body", "foo bar"),
             client().prepareIndex("test").setId("4").setSource("body", "foo baz bar")
         );
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> query 1");
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar").minimumShouldMatch("2")).get();
@@ -235,6 +237,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
             client().prepareIndex("test").setId("7").setSource("body2", "foo bar", "other", "foo"),
             client().prepareIndex("test").setId("8").setSource("body2", "foo baz bar", "other", "foo")
         );
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> query 5");
         searchResponse = client().prepareSearch()
@@ -256,7 +259,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
         assertSearchHits(searchResponse, "6", "7", "8");
     }
 
-    public void testNestedFieldSimpleQueryString() throws IOException {
+    public void testNestedFieldSimpleQueryString() throws Exception {
         assertAcked(
             prepareCreate("test").setMapping(
                 jsonBuilder().startObject()
@@ -275,6 +278,7 @@ public void testNestedFieldSimpleQueryString() throws IOException {
         );
         client().prepareIndex("test").setId("1").setSource("body", "foo bar baz").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar baz").field("body")).get();
         assertHitCount(searchResponse, 1L);
@@ -359,6 +363,8 @@ public void testSimpleQueryStringLenient() throws ExecutionException, Interrupte
             client().prepareIndex("test2").setId("10").setSource("field", 5)
         );
         refresh();
+        indexRandomForConcurrentSearch("test1");
+        indexRandomForConcurrentSearch("test2");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setAllowPartialSearchResults(true)
@@ -419,6 +425,7 @@ public void testSimpleQueryStringUsesFieldAnalyzer() throws Exception {
         client().prepareIndex("test").setId("2").setSource("foo", 234, "bar", "bcd").get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("123").field("foo").field("bar")).get();
         assertHitCount(searchResponse, 1L);
@@ -469,6 +476,7 @@ public void testBasicAllQuery() throws Exception {
         reqs.add(client().prepareIndex("test").setId("2").setSource("f2", "Bar"));
         reqs.add(client().prepareIndex("test").setId("3").setSource("f3", "foo bar baz"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
         assertHitCount(resp, 2L);
@@ -492,6 +500,7 @@ public void testWithDate() throws Exception {
         reqs.add(client().prepareIndex("test").setId("1").setSource("f1", "foo", "f_date", "2015/09/02"));
         reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
         assertHits(resp.getHits(), "1", "2");
@@ -523,6 +532,7 @@ public void testWithLotsOfTypes() throws Exception {
             client().prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01", "f_float", "1.8", "f_ip", "127.0.0.2")
         );
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
         assertHits(resp.getHits(), "1", "2");
@@ -550,6 +560,7 @@ public void testDocWithAllTypes() throws Exception {
         String docBody = copyToStringFromClasspath("/org/opensearch/search/query/all-example-document.json");
         reqs.add(client().prepareIndex("test").setId("1").setSource(docBody, MediaTypeRegistry.JSON));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
         assertHits(resp.getHits(), "1");
@@ -596,6 +607,7 @@ public void testKeywordWithWhitespace() throws Exception {
         reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar"));
         reqs.add(client().prepareIndex("test").setId("3").setSource("f1", "foo bar"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
         assertHits(resp.getHits(), "3");
@@ -663,6 +675,7 @@ public void testFieldAlias() throws Exception {
         indexRequests.add(client().prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two"));
         indexRequests.add(client().prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_alias")).get();
 
@@ -681,6 +694,7 @@ public void testFieldAliasWithWildcardField() throws Exception {
         indexRequests.add(client().prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two"));
         indexRequests.add(client().prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_*")).get();
 
@@ -697,6 +711,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
         List indexRequests = new ArrayList<>();
         indexRequests.add(client().prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         // The wildcard field matches aliases for both a text and boolean field.
         // By default, the boolean field should be ignored when building the query.

From c1962cc7c6a9389cc31d28befd65298ac4f003ad Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Sun, 5 Nov 2023 20:47:43 -0800
Subject: [PATCH 271/388] Fix flaky pit/scroll tests in SegmentReplicationIT
 (#10770)

This change adds an assertBusy to wait until files are cleared from disk as it is not
synchronous with the scroll/pit removal.

Signed-off-by: Marc Handalian 
---
 .../replication/SegmentReplicationIT.java      | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
index a2996d87a851b..9c93a8f85db8e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
@@ -1066,9 +1066,14 @@ public void testScrollCreatedOnReplica() throws Exception {
 
         client(replica).prepareClearScroll().addScrollId(searchResponse.getScrollId()).get();
 
-        currentFiles = List.of(replicaShard.store().directory().listAll());
-        assertFalse("Files should be cleaned up post scroll clear request", currentFiles.containsAll(snapshottedSegments));
+        assertBusy(
+            () -> assertFalse(
+                "Files should be cleaned up post scroll clear request",
+                List.of(replicaShard.store().directory().listAll()).containsAll(snapshottedSegments)
+            )
+        );
         assertEquals(100, scrollHits);
+
     }
 
     /**
@@ -1327,9 +1332,12 @@ public void testPitCreatedOnReplica() throws Exception {
         // delete the PIT
         DeletePitRequest deletePITRequest = new DeletePitRequest(pitResponse.getId());
         client().execute(DeletePitAction.INSTANCE, deletePITRequest).actionGet();
-
-        currentFiles = List.of(replicaShard.store().directory().listAll());
-        assertFalse("Files should be cleaned up", currentFiles.containsAll(snapshottedSegments));
+        assertBusy(
+            () -> assertFalse(
+                "Files should be cleaned up",
+                List.of(replicaShard.store().directory().listAll()).containsAll(snapshottedSegments)
+            )
+        );
     }
 
     /**

From 1782b906a0a9bf293bbc8e7f33058153790827dc Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Mon, 6 Nov 2023 12:21:05 -0500
Subject: [PATCH 272/388] [Streaming Indexing] Introduce new experimental HTTP
 transport based on Netty 4 and Project Reactor (Reactor Netty) (#9672)

* [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty)

Signed-off-by: Andriy Redko 

* Remove HttpChunk for now since it is not used in non-streaming APIs

Signed-off-by: Andriy Redko 

* Fix postmerge issues

Signed-off-by: Andriy Redko 

---------

Signed-off-by: Andriy Redko 
---
 CHANGELOG.md                                  |   1 +
 buildSrc/version.properties                   |   4 +
 .../netty4/Netty4HttpServerTransport.java     |  14 +
 .../licenses/reactive-streams-1.0.4.jar.sha1  |   1 -
 .../licenses/reactive-streams-LICENSE.txt     |  21 -
 plugins/repository-azure/build.gradle         |  22 +-
 .../licenses/reactive-streams-1.0.4.jar.sha1  |   1 -
 .../licenses/reactive-streams-LICENSE.txt     |  21 -
 .../licenses/reactor-core-3.5.6.jar.sha1      |   1 -
 .../licenses/reactor-netty-1.1.8.jar.sha1     |   1 -
 .../reactor-netty-core-1.1.12.jar.sha1        |   1 +
 .../reactor-netty-core-1.1.8.jar.sha1         |   1 -
 .../reactor-netty-http-1.1.12.jar.sha1        |   1 +
 .../reactor-netty-http-1.1.9.jar.sha1         |   1 -
 plugins/repository-s3/build.gradle            |   1 -
 .../licenses/reactive-streams-1.0.4.jar.sha1  |   1 -
 .../licenses/reactive-streams-LICENSE.txt     |  21 -
 .../licenses/reactive-streams-NOTICE.txt      |   0
 plugins/transport-reactor-netty4/build.gradle | 264 ++++++++
 .../licenses/netty-LICENSE.txt                | 202 ++++++
 .../licenses/netty-NOTICE.txt                 | 116 ++++
 .../netty-buffer-4.1.100.Final.jar.sha1       |   1 +
 .../netty-codec-4.1.100.Final.jar.sha1        |   1 +
 .../netty-codec-dns-4.1.100.Final.jar.sha1    |   1 +
 .../netty-codec-http-4.1.100.Final.jar.sha1   |   1 +
 .../netty-codec-http2-4.1.100.Final.jar.sha1  |   1 +
 .../netty-common-4.1.100.Final.jar.sha1       |   1 +
 .../netty-handler-4.1.100.Final.jar.sha1      |   1 +
 .../netty-resolver-4.1.100.Final.jar.sha1     |   1 +
 .../netty-resolver-dns-4.1.100.Final.jar.sha1 |   1 +
 .../netty-transport-4.1.100.Final.jar.sha1    |   1 +
 ...-native-unix-common-4.1.100.Final.jar.sha1 |   1 +
 .../licenses/reactor-LICENSE.txt              | 201 ++++++
 .../licenses/reactor-NOTICE.txt}              |   0
 .../reactor-netty-core-1.1.12.jar.sha1        |   1 +
 .../reactor-netty-http-1.1.12.jar.sha1        |   1 +
 .../OpenSearchReactorNetty4IntegTestCase.java |  73 +++
 .../ReactorNetty4HttpRequestSizeLimitIT.java  | 159 +++++
 .../netty4/ReactorNetty4PipeliningIT.java     |  68 ++
 .../reactor/netty4/HttpConversionUtil.java    |  47 ++
 .../netty4/NonStreamingHttpChannel.java       |  76 +++
 .../netty4/NonStreamingRequestConsumer.java   | 104 ++++
 .../netty4/ReactorNetty4HttpRequest.java      | 272 ++++++++
 .../netty4/ReactorNetty4HttpResponse.java     |  42 ++
 .../ReactorNetty4HttpServerChannel.java       |  53 ++
 .../ReactorNetty4HttpServerTransport.java     | 313 ++++++++++
 .../http/reactor/netty4/package-info.java     |  12 +
 .../reactor/ReactorNetty4Plugin.java          | 109 ++++
 .../transport/reactor/SharedGroupFactory.java | 164 +++++
 .../transport/reactor/netty4/Netty4Utils.java | 142 +++++
 .../netty4/ReactorNetty4Transport.java        |  35 ++
 .../reactor/netty4/package-info.java          |  12 +
 .../transport/reactor/package-info.java       |  12 +
 .../plugin-metadata/plugin-security.policy    |  24 +
 .../reactor/netty4/ReactorHttpClient.java     | 208 +++++++
 .../netty4/ReactorNetty4BadRequestTests.java  | 122 ++++
 ...ReactorNetty4HttpServerTransportTests.java | 579 ++++++++++++++++++
 server/build.gradle                           |   8 +-
 .../licenses/reactive-streams-1.0.4.jar.sha1  |   0
 .../licenses/reactive-streams-LICENSE.txt     |   0
 .../licenses/reactive-streams-NOTICE.txt      |   0
 server/licenses/reactor-LICENSE.txt           | 201 ++++++
 .../licenses/reactor-NOTICE.txt               |   0
 server/licenses/reactor-core-3.5.11.jar.sha1  |   1 +
 .../common/settings/ClusterSettings.java      |   1 +
 .../http/HttpTransportSettings.java           |   8 +
 .../opensearch/test/OpenSearchTestCase.java   |   3 +
 67 files changed, 3668 insertions(+), 90 deletions(-)
 delete mode 100644 plugins/discovery-ec2/licenses/reactive-streams-1.0.4.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/reactive-streams-LICENSE.txt
 delete mode 100644 plugins/repository-azure/licenses/reactive-streams-1.0.4.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/reactive-streams-LICENSE.txt
 delete mode 100644 plugins/repository-azure/licenses/reactor-core-3.5.6.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.12.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.12.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.9.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/reactive-streams-1.0.4.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/reactive-streams-LICENSE.txt
 delete mode 100644 plugins/repository-s3/licenses/reactive-streams-NOTICE.txt
 create mode 100644 plugins/transport-reactor-netty4/build.gradle
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-LICENSE.txt
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-NOTICE.txt
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-LICENSE.txt
 rename plugins/{crypto-kms/licenses/reactive-streams-NOTICE.txt => transport-reactor-netty4/licenses/reactor-NOTICE.txt} (100%)
 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.12.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.12.jar.sha1
 create mode 100644 plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/OpenSearchReactorNetty4IntegTestCase.java
 create mode 100644 plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequestSizeLimitIT.java
 create mode 100644 plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4PipeliningIT.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/HttpConversionUtil.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingHttpChannel.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingRequestConsumer.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequest.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpResponse.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerChannel.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/package-info.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/SharedGroupFactory.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/Netty4Utils.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/ReactorNetty4Transport.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/package-info.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/package-info.java
 create mode 100644 plugins/transport-reactor-netty4/src/main/plugin-metadata/plugin-security.policy
 create mode 100644 plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java
 create mode 100644 plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4BadRequestTests.java
 create mode 100644 plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransportTests.java
 rename {plugins/crypto-kms => server}/licenses/reactive-streams-1.0.4.jar.sha1 (100%)
 rename {plugins/crypto-kms => server}/licenses/reactive-streams-LICENSE.txt (100%)
 rename {plugins/discovery-ec2 => server}/licenses/reactive-streams-NOTICE.txt (100%)
 create mode 100644 server/licenses/reactor-LICENSE.txt
 rename plugins/repository-azure/licenses/reactive-streams-NOTICE.txt => server/licenses/reactor-NOTICE.txt (100%)
 create mode 100644 server/licenses/reactor-core-3.5.11.jar.sha1

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b8f1084eafd4..a71d2ff537834 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -100,6 +100,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Make index and global metadata upload timeout dynamic cluster settings ([#10814](https://github.com/opensearch-project/OpenSearch/pull/10814))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
 - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
+- [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 0d98cba35448f..f19437979c852 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -29,6 +29,10 @@ jna               = 5.13.0
 netty             = 4.1.100.Final
 joda              = 2.12.2
 
+# project reactor
+reactor_netty     = 1.1.12
+reactor           = 3.5.11
+
 # client dependencies
 httpclient5       = 5.2.1
 httpcore5         = 5.2.2
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java
index 1677f333a4b1c..4970c42163ac3 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java
@@ -116,6 +116,9 @@
 import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
 import static org.opensearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
 
+/**
+ * The HTTP transport implementations based on Netty 4.
+ */
 public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
     private static final Logger logger = LogManager.getLogger(Netty4HttpServerTransport.class);
 
@@ -184,6 +187,17 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
     private volatile ServerBootstrap serverBootstrap;
     private volatile SharedGroupFactory.SharedGroup sharedGroup;
 
+    /**
+     * Creates new HTTP transport implementations based on Netty 4
+     * @param settings seetings
+     * @param networkService network service
+     * @param bigArrays big array allocator
+     * @param threadPool thread pool instance
+     * @param xContentRegistry XContent registry instance
+     * @param dispatcher dispatcher instance
+     * @param clusterSettings cluster settings
+     * @param sharedGroupFactory shared group factory
+     */
     public Netty4HttpServerTransport(
         Settings settings,
         NetworkService networkService,
diff --git a/plugins/discovery-ec2/licenses/reactive-streams-1.0.4.jar.sha1 b/plugins/discovery-ec2/licenses/reactive-streams-1.0.4.jar.sha1
deleted file mode 100644
index 45a80e3f7e361..0000000000000
--- a/plugins/discovery-ec2/licenses/reactive-streams-1.0.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3864a1320d97d7b045f729a326e1e077661f31b7
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/reactive-streams-LICENSE.txt b/plugins/discovery-ec2/licenses/reactive-streams-LICENSE.txt
deleted file mode 100644
index 1e3c7e7c77495..0000000000000
--- a/plugins/discovery-ec2/licenses/reactive-streams-LICENSE.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-MIT No Attribution
-
-Copyright 2014 Reactive Streams
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
\ No newline at end of file
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 26e2b4813b8a5..51f2057b4bedb 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -56,11 +56,8 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
-  api 'org.reactivestreams:reactive-streams:1.0.4'
-  api 'io.projectreactor:reactor-core:3.5.6'
-  api 'io.projectreactor.netty:reactor-netty:1.1.8'
-  api 'io.projectreactor.netty:reactor-netty-core:1.1.8'
-  api 'io.projectreactor.netty:reactor-netty-http:1.1.9'
+  api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
+  api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
@@ -101,10 +98,6 @@ thirdPartyAudit {
     'com.azure.storage.internal.avro.implementation.schema.AvroSchema',
     'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
     'io.micrometer.context.ContextAccessor',
-    'io.micrometer.context.ContextRegistry',
-    'io.micrometer.context.ContextSnapshot',
-    'io.micrometer.context.ContextSnapshot$Scope',
-    'io.micrometer.core.instrument.Clock',
     'io.micrometer.core.instrument.Counter',
     'io.micrometer.core.instrument.Counter$Builder',
     'io.micrometer.core.instrument.DistributionSummary',
@@ -114,14 +107,10 @@ thirdPartyAudit {
     'io.micrometer.core.instrument.Meter',
     'io.micrometer.core.instrument.MeterRegistry',
     'io.micrometer.core.instrument.Metrics',
-    'io.micrometer.core.instrument.Tag',
-    'io.micrometer.core.instrument.Tags',
     'io.micrometer.core.instrument.Timer',
     'io.micrometer.core.instrument.Timer$Builder',
     'io.micrometer.core.instrument.Timer$Sample',
-    'io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics',
     'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
-    'io.micrometer.core.instrument.search.Search',
     'io.netty.channel.epoll.Epoll',
     'io.netty.channel.epoll.EpollDatagramChannel',
     'io.netty.channel.epoll.EpollServerSocketChannel',
@@ -168,9 +157,6 @@ thirdPartyAudit {
     'org.slf4j.impl.StaticLoggerBinder',
     'org.slf4j.impl.StaticMDCBinder',
     'org.slf4j.impl.StaticMarkerBinder',
-    'reactor.blockhound.BlockHound$Builder',
-    'reactor.blockhound.integration.BlockHoundIntegration',
-    'io.micrometer.context.ThreadLocalAccessor',
     'io.micrometer.common.KeyValue',
     'io.micrometer.common.KeyValues',
     'io.micrometer.common.docs.KeyName',
@@ -190,6 +176,7 @@ thirdPartyAudit {
     'io.micrometer.tracing.handler.PropagatingSenderTracingObservationHandler',
     'io.micrometer.tracing.propagation.Propagator',
     'io.micrometer.core.instrument.observation.MeterObservationHandler',
+    'io.micrometer.core.instrument.Tags',
     'io.micrometer.observation.ObservationHandler',
     'io.micrometer.observation.ObservationRegistry',
     'io.micrometer.observation.ObservationRegistry$ObservationConfig',
@@ -210,8 +197,7 @@ thirdPartyAudit {
     'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
-    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
-    'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1'
   )
 }
 
diff --git a/plugins/repository-azure/licenses/reactive-streams-1.0.4.jar.sha1 b/plugins/repository-azure/licenses/reactive-streams-1.0.4.jar.sha1
deleted file mode 100644
index 45a80e3f7e361..0000000000000
--- a/plugins/repository-azure/licenses/reactive-streams-1.0.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3864a1320d97d7b045f729a326e1e077661f31b7
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactive-streams-LICENSE.txt b/plugins/repository-azure/licenses/reactive-streams-LICENSE.txt
deleted file mode 100644
index 1e3c7e7c77495..0000000000000
--- a/plugins/repository-azure/licenses/reactive-streams-LICENSE.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-MIT No Attribution
-
-Copyright 2014 Reactive Streams
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-core-3.5.6.jar.sha1 b/plugins/repository-azure/licenses/reactor-core-3.5.6.jar.sha1
deleted file mode 100644
index ad9b7263e7b38..0000000000000
--- a/plugins/repository-azure/licenses/reactor-core-3.5.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-027fdc551537b349389176a23a192f11a7a3d7de
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1
deleted file mode 100644
index 6b6bf1903b16c..0000000000000
--- a/plugins/repository-azure/licenses/reactor-netty-1.1.8.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d53a9d7d0395285f4c81664494fcd61477626e32
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.12.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.12.jar.sha1
new file mode 100644
index 0000000000000..352d69396d0c9
--- /dev/null
+++ b/plugins/repository-azure/licenses/reactor-netty-core-1.1.12.jar.sha1
@@ -0,0 +1 @@
+378dc5a375e6440099e837b22cf4b01341cbe4ea
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1
deleted file mode 100644
index 707631f4dfe0c..0000000000000
--- a/plugins/repository-azure/licenses/reactor-netty-core-1.1.8.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-48999c4ae27cdcee5eaff9dfd150a8b64624f0f5
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.12.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.12.jar.sha1
new file mode 100644
index 0000000000000..1bcb0e0c52950
--- /dev/null
+++ b/plugins/repository-azure/licenses/reactor-netty-http-1.1.12.jar.sha1
@@ -0,0 +1 @@
+e839fadb8f45d8a7a2783466faedd03373366c23
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.9.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.9.jar.sha1
deleted file mode 100644
index 96deead2c75d1..0000000000000
--- a/plugins/repository-azure/licenses/reactor-netty-http-1.1.9.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-408b3037133f2e8ab0f195ccd3f807026be9b860
\ No newline at end of file
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 44fd45b265e82..560d12d14395d 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -70,7 +70,6 @@ dependencies {
   api "software.amazon.awssdk:sts:${versions.aws}"
   api "software.amazon.awssdk:netty-nio-client:${versions.aws}"
 
-  api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
diff --git a/plugins/repository-s3/licenses/reactive-streams-1.0.4.jar.sha1 b/plugins/repository-s3/licenses/reactive-streams-1.0.4.jar.sha1
deleted file mode 100644
index 45a80e3f7e361..0000000000000
--- a/plugins/repository-s3/licenses/reactive-streams-1.0.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3864a1320d97d7b045f729a326e1e077661f31b7
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/reactive-streams-LICENSE.txt b/plugins/repository-s3/licenses/reactive-streams-LICENSE.txt
deleted file mode 100644
index 1e3c7e7c77495..0000000000000
--- a/plugins/repository-s3/licenses/reactive-streams-LICENSE.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-MIT No Attribution
-
-Copyright 2014 Reactive Streams
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/reactive-streams-NOTICE.txt b/plugins/repository-s3/licenses/reactive-streams-NOTICE.txt
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/plugins/transport-reactor-netty4/build.gradle b/plugins/transport-reactor-netty4/build.gradle
new file mode 100644
index 0000000000000..7d7eb330b4a55
--- /dev/null
+++ b/plugins/transport-reactor-netty4/build.gradle
@@ -0,0 +1,264 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import org.opensearch.gradle.info.BuildParams
+import org.opensearch.gradle.test.RestIntegTestTask
+import org.opensearch.gradle.test.TestTask
+import org.opensearch.gradle.test.rest.JavaRestTestPlugin
+import org.opensearch.gradle.test.InternalClusterTestPlugin
+
+apply plugin: 'opensearch.yaml-rest-test'
+apply plugin: 'opensearch.java-rest-test'
+apply plugin: 'opensearch.internal-cluster-test'
+
+// The transport-reactor-netty4 plugin is published to maven
+apply plugin: 'opensearch.publish'
+
+opensearchplugin {
+  description 'Reactor Netty 4 based transport implementation'
+  classname 'org.opensearch.transport.reactor.ReactorNetty4Plugin'
+  hasClientJar = true
+}
+
+dependencies {
+  // network stack
+  api "io.netty:netty-buffer:${versions.netty}"
+  api "io.netty:netty-codec:${versions.netty}"
+  api "io.netty:netty-codec-dns:${versions.netty}"
+  api "io.netty:netty-codec-http:${versions.netty}"
+  api "io.netty:netty-codec-http2:${versions.netty}"
+  api "io.netty:netty-common:${versions.netty}"
+  api "io.netty:netty-handler:${versions.netty}"
+  api "io.netty:netty-resolver-dns:${versions.netty}"
+  api "io.netty:netty-resolver:${versions.netty}"
+  api "io.netty:netty-transport:${versions.netty}"
+  api "io.netty:netty-transport-native-unix-common:${versions.netty}"
+
+  api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}"
+  api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
+
+  testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+  testImplementation "io.projectreactor:reactor-test:${versions.reactor}"
+  testImplementation project(":modules:transport-netty4")
+}
+
+restResources {
+  restApi {
+    includeCore '_common', 'cluster', 'nodes'
+  }
+}
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /netty-.*/, to: 'netty'
+  mapping from: /reactor-.*/, to: 'reactor'
+}
+
+// TODO: Remove that once we have a complete test suite
+testingConventions.enabled = false
+
+test {
+  /*
+   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+   * other if we allow them to set the number of available processors as it's set-once in Netty.
+   */
+  systemProperty 'opensearch.set.netty.runtime.available.processors', 'false'
+}
+
+internalClusterTest {
+  systemProperty 'opensearch.set.netty.runtime.available.processors', 'false'
+}
+
+javaRestTest {
+  systemProperty 'opensearch.set.netty.runtime.available.processors', 'false'
+}
+
+thirdPartyAudit {
+  ignoreMissingClasses(
+    'com.aayushatharva.brotli4j.Brotli4jLoader',
+    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
+    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
+    'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
+    // classes are missing
+
+    // from io.netty.logging.CommonsLoggerFactory (netty)
+    'org.apache.commons.logging.Log',
+    'org.apache.commons.logging.LogFactory',
+
+    // from Log4j (deliberate, Netty will fallback to Log4j 2)
+    'org.apache.log4j.Level',
+    'org.apache.log4j.Logger',
+
+    // from io.netty.handler.ssl.OpenSslEngine (netty)
+    'io.netty.internal.tcnative.Buffer',
+    'io.netty.internal.tcnative.CertificateCompressionAlgo',
+    'io.netty.internal.tcnative.Library',
+    'io.netty.internal.tcnative.SSL',
+    'io.netty.internal.tcnative.SSLContext',
+    'io.netty.internal.tcnative.SSLPrivateKeyMethod',
+
+    // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
+    'org.bouncycastle.cert.X509v3CertificateBuilder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
+    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
+    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
+
+    // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
+    'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
+    'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
+    'org.eclipse.jetty.npn.NextProtoNego',
+
+    // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
+    'org.jboss.marshalling.ByteInput',
+
+    // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
+    'org.jboss.marshalling.ByteOutput',
+
+    // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
+    'org.jboss.marshalling.Marshaller',
+
+    // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
+    'org.jboss.marshalling.MarshallerFactory',
+    'org.jboss.marshalling.MarshallingConfiguration',
+    'org.jboss.marshalling.Unmarshaller',
+
+    // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional
+    'org.slf4j.helpers.FormattingTuple',
+    'org.slf4j.helpers.MessageFormatter',
+    'org.slf4j.Logger',
+    'org.slf4j.LoggerFactory',
+    'org.slf4j.spi.LocationAwareLogger',
+
+    'com.google.protobuf.nano.CodedOutputByteBufferNano',
+    'com.google.protobuf.nano.MessageNano',
+    'com.ning.compress.BufferRecycler',
+    'com.ning.compress.lzf.ChunkDecoder',
+    'com.ning.compress.lzf.ChunkEncoder',
+    'com.ning.compress.lzf.LZFChunk',
+    'com.ning.compress.lzf.LZFEncoder',
+    'com.ning.compress.lzf.util.ChunkDecoderFactory',
+    'com.ning.compress.lzf.util.ChunkEncoderFactory',
+    'lzma.sdk.lzma.Encoder',
+    'net.jpountz.lz4.LZ4Compressor',
+    'net.jpountz.lz4.LZ4Factory',
+    'net.jpountz.lz4.LZ4FastDecompressor',
+    'net.jpountz.xxhash.XXHash32',
+    'net.jpountz.xxhash.XXHashFactory',
+    'io.netty.internal.tcnative.AsyncSSLPrivateKeyMethod',
+    'io.netty.internal.tcnative.AsyncTask',
+    'io.netty.internal.tcnative.CertificateCallback',
+    'io.netty.internal.tcnative.CertificateVerifier',
+    'io.netty.internal.tcnative.ResultCallback',
+    'io.netty.internal.tcnative.SessionTicketKey',
+    'io.netty.internal.tcnative.SniHostNameMatcher',
+    'io.netty.internal.tcnative.SSL',
+    'io.netty.internal.tcnative.SSLSession',
+    'io.netty.internal.tcnative.SSLSessionCache',
+    'io.netty.channel.epoll.Epoll',
+    'io.netty.channel.epoll.EpollDatagramChannel',
+    'io.netty.channel.epoll.EpollServerSocketChannel',
+    'io.netty.channel.epoll.EpollSocketChannel',
+    'io.netty.channel.kqueue.KQueue',
+    'io.netty.channel.kqueue.KQueueDatagramChannel',
+    'io.netty.channel.kqueue.KQueueServerSocketChannel',
+    'io.netty.channel.kqueue.KQueueSocketChannel',
+    'io.netty.handler.codec.haproxy.HAProxyMessage',
+    'io.netty.handler.codec.haproxy.HAProxyMessageDecoder',
+    'io.netty.handler.proxy.ProxyHandler',
+    'io.netty.incubator.channel.uring.IOUring',
+    'io.netty.incubator.channel.uring.IOUringDatagramChannel',
+    'io.netty.incubator.channel.uring.IOUringServerSocketChannel',
+    'io.netty.incubator.channel.uring.IOUringSocketChannel',
+
+    'org.eclipse.jetty.alpn.ALPN$ClientProvider',
+    'org.eclipse.jetty.alpn.ALPN$ServerProvider',
+    'org.eclipse.jetty.alpn.ALPN',
+
+    'org.conscrypt.AllocatedBuffer',
+    'org.conscrypt.BufferAllocator',
+    'org.conscrypt.Conscrypt',
+    'org.conscrypt.HandshakeListener',
+
+    'reactor.blockhound.BlockHound$Builder',
+    'reactor.blockhound.integration.BlockHoundIntegration',
+
+    'io.micrometer.common.KeyValue',
+    'io.micrometer.common.KeyValues',
+    'io.micrometer.common.docs.KeyName',
+    'io.micrometer.context.ContextAccessor',
+    'io.micrometer.core.instrument.Counter',
+    'io.micrometer.core.instrument.Counter$Builder',
+    'io.micrometer.core.instrument.DistributionSummary',
+    'io.micrometer.core.instrument.DistributionSummary$Builder',
+    'io.micrometer.core.instrument.Gauge',
+    'io.micrometer.core.instrument.Gauge$Builder',
+    'io.micrometer.core.instrument.Meter',
+    'io.micrometer.core.instrument.Meter$Type',
+    'io.micrometer.core.instrument.MeterRegistry',
+    'io.micrometer.core.instrument.Metrics',
+    'io.micrometer.core.instrument.Tags',
+    'io.micrometer.core.instrument.Timer',
+    'io.micrometer.core.instrument.Timer$Builder',
+    'io.micrometer.core.instrument.Timer$Sample',
+    'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
+    'io.micrometer.core.instrument.docs.MeterDocumentation',
+    'io.micrometer.core.instrument.observation.MeterObservationHandler',
+    'io.micrometer.observation.Observation',
+    'io.micrometer.observation.Observation$Context',
+    'io.micrometer.observation.ObservationHandler',
+    'io.micrometer.observation.ObservationRegistry',
+    'io.micrometer.observation.ObservationRegistry$ObservationConfig',
+    'io.micrometer.observation.docs.ObservationDocumentation',
+    'io.micrometer.observation.transport.ReceiverContext',
+    'io.micrometer.observation.transport.RequestReplyReceiverContext',
+    'io.micrometer.observation.transport.RequestReplySenderContext',
+    'io.micrometer.observation.transport.SenderContext',
+    'io.micrometer.tracing.Span',
+    'io.micrometer.tracing.Tracer',
+    'io.micrometer.tracing.docs.SpanDocumentation',
+    'io.micrometer.tracing.handler.DefaultTracingObservationHandler',
+    'io.micrometer.tracing.handler.PropagatingReceiverTracingObservationHandler',
+    'io.micrometer.tracing.handler.PropagatingSenderTracingObservationHandler',
+    'io.micrometer.tracing.propagation.Propagator'
+  )
+
+  ignoreViolations(
+    'io.netty.util.internal.PlatformDependent0',
+    'io.netty.util.internal.PlatformDependent0$1',
+    'io.netty.util.internal.PlatformDependent0$2',
+    'io.netty.util.internal.PlatformDependent0$3',
+    'io.netty.util.internal.PlatformDependent0$4',
+    'io.netty.util.internal.PlatformDependent0$6',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
+    'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5'
+  )
+}
diff --git a/plugins/transport-reactor-netty4/licenses/netty-LICENSE.txt b/plugins/transport-reactor-netty4/licenses/netty-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/transport-reactor-netty4/licenses/netty-NOTICE.txt b/plugins/transport-reactor-netty4/licenses/netty-NOTICE.txt
new file mode 100644
index 0000000000000..5bbf91a14de23
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-NOTICE.txt
@@ -0,0 +1,116 @@
+
+                            The Netty Project
+                            =================
+
+Please visit the Netty web site for more information:
+
+  * http://netty.io/
+
+Copyright 2011 The Netty Project
+
+The Netty Project licenses this file to you under the Apache License,
+version 2.0 (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+
+Also, please refer to each LICENSE..txt file, which is located in
+the 'license' directory of the distribution file, for the license terms of the
+components that this product depends on.
+
+-------------------------------------------------------------------------------
+This product contains the extensions to Java Collections Framework which has
+been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
+
+  * LICENSE:
+    * license/LICENSE.jsr166y.txt (Public Domain)
+  * HOMEPAGE:
+    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
+    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
+
+This product contains a modified version of Robert Harder's Public Domain
+Base64 Encoder and Decoder, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.base64.txt (Public Domain)
+  * HOMEPAGE:
+    * http://iharder.sourceforge.net/current/java/base64/
+
+This product contains a modified version of 'JZlib', a re-implementation of
+zlib in pure Java, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.jzlib.txt (BSD Style License)
+  * HOMEPAGE:
+    * http://www.jcraft.com/jzlib/
+
+This product contains a modified version of 'Webbit', a Java event based
+WebSocket and HTTP server:
+
+  * LICENSE:
+    * license/LICENSE.webbit.txt (BSD License)
+  * HOMEPAGE:
+    * https://github.com/joewalnes/webbit
+
+This product optionally depends on 'Protocol Buffers', Google's data
+interchange format, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.protobuf.txt (New BSD License)
+  * HOMEPAGE:
+    * http://code.google.com/p/protobuf/
+
+This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
+a temporary self-signed X.509 certificate when the JVM does not provide the
+equivalent functionality.  It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.bouncycastle.txt (MIT License)
+  * HOMEPAGE:
+    * http://www.bouncycastle.org/
+
+This product optionally depends on 'SLF4J', a simple logging facade for Java,
+which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.slf4j.txt (MIT License)
+  * HOMEPAGE:
+    * http://www.slf4j.org/
+
+This product optionally depends on 'Apache Commons Logging', a logging
+framework, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.commons-logging.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://commons.apache.org/logging/
+
+This product optionally depends on 'Apache Log4J', a logging framework,
+which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.log4j.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://logging.apache.org/log4j/
+
+This product optionally depends on 'JBoss Logging', a logging framework,
+which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1)
+  * HOMEPAGE:
+    * http://anonsvn.jboss.org/repos/common/common-logging-spi/
+
+This product optionally depends on 'Apache Felix', an open source OSGi
+framework implementation, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.felix.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://felix.apache.org/
diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..aaf2e35302d77
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+39b05d2d4027971bf99111a9be1d7035a116bb55
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..a77333ea8ae47
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+9c3c71e7cf3b8ce3bfc9fa52a524b9ca7ddf259c
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..dfa4a0fbea94c
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+663b1b7bf3ff0f12fde4df20c72d9e94584ebffa
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..6f26bf4e6a9b5
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+992623e7d8f2d96e41faf1687bb963f5433e3517
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..bf5605151406e
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+cbf1a430ea44dbdedbcde16b185cbb95f28d72c7
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..d2ff72db60d1f
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+847f942381145de23f21c836d05b0677474271d3
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..f12a6046e96d0
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+4c0acdb8bb73647ebb3847ac2d503d53d72c02b4
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..8e4179ba15942
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+fe62f9ccd41b8660d07639dbbab8ae1edd6f2720
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..0948daa05fff6
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+62dbdf5f25eda75ea8456be1ed72b3fcb0d18774
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..ab2819da570fd
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+6620fbfb47667a5eb6050e35c7b4c88000bcd77f
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
new file mode 100644
index 0000000000000..30d7758302e37
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.100.Final.jar.sha1
@@ -0,0 +1 @@
+648ff5571022dbfa6789122e3872477bbf67fa7b
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-LICENSE.txt b/plugins/transport-reactor-netty4/licenses/reactor-LICENSE.txt
new file mode 100644
index 0000000000000..e5583c184e67a
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/reactor-LICENSE.txt
@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/crypto-kms/licenses/reactive-streams-NOTICE.txt b/plugins/transport-reactor-netty4/licenses/reactor-NOTICE.txt
similarity index 100%
rename from plugins/crypto-kms/licenses/reactive-streams-NOTICE.txt
rename to plugins/transport-reactor-netty4/licenses/reactor-NOTICE.txt
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.12.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.12.jar.sha1
new file mode 100644
index 0000000000000..352d69396d0c9
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.12.jar.sha1
@@ -0,0 +1 @@
+378dc5a375e6440099e837b22cf4b01341cbe4ea
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.12.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.12.jar.sha1
new file mode 100644
index 0000000000000..1bcb0e0c52950
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.12.jar.sha1
@@ -0,0 +1 @@
+e839fadb8f45d8a7a2783466faedd03373366c23
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/OpenSearchReactorNetty4IntegTestCase.java b/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/OpenSearchReactorNetty4IntegTestCase.java
new file mode 100644
index 0000000000000..abbd50bf1b235
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/OpenSearchReactorNetty4IntegTestCase.java
@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch;
+
+import org.opensearch.common.network.NetworkModule;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.transport.Netty4ModulePlugin;
+import org.opensearch.transport.reactor.ReactorNetty4Plugin;
+import org.opensearch.transport.reactor.netty4.ReactorNetty4Transport;
+
+import java.util.Collection;
+import java.util.List;
+
+public abstract class OpenSearchReactorNetty4IntegTestCase extends OpenSearchIntegTestCase {
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @Override
+    protected boolean addMockTransportService() {
+        return false;
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+        // randomize netty settings
+        if (randomBoolean()) {
+            builder.put(ReactorNetty4Transport.SETTING_WORKER_COUNT.getKey(), random().nextInt(3) + 1);
+        }
+        builder.put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4ModulePlugin.NETTY_TRANSPORT_NAME);
+        builder.put(NetworkModule.HTTP_TYPE_KEY, ReactorNetty4Plugin.REACTOR_NETTY_HTTP_TRANSPORT_NAME);
+        return builder.build();
+    }
+
+    @Override
+    protected Collection> nodePlugins() {
+        return List.of(ReactorNetty4Plugin.class, Netty4ModulePlugin.class);
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequestSizeLimitIT.java b/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequestSizeLimitIT.java
new file mode 100644
index 0000000000000..833d60375a2bd
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequestSizeLimitIT.java
@@ -0,0 +1,159 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.OpenSearchReactorNetty4IntegTestCase;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
+import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
+import org.opensearch.test.OpenSearchIntegTestCase.Scope;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.util.ReferenceCounted;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass.
+ *
+ * As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing
+ * a single node "cluster".
+ */
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1)
+public class ReactorNetty4HttpRequestSizeLimitIT extends OpenSearchReactorNetty4IntegTestCase {
+
+    private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB);
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), LIMIT)
+            .build();
+    }
+
+    public void testLimitsInFlightRequests() throws Exception {
+        ensureGreen();
+
+        // we use the limit size as a (very) rough indication on how many requests we should sent to hit the limit
+        int numRequests = LIMIT.bytesAsInt() / 100;
+
+        StringBuilder bulkRequest = new StringBuilder();
+        for (int i = 0; i < numRequests; i++) {
+            bulkRequest.append("{\"index\": {}}");
+            bulkRequest.append(System.lineSeparator());
+            bulkRequest.append("{ \"field\" : \"value\" }");
+            bulkRequest.append(System.lineSeparator());
+        }
+
+        List> requests = new ArrayList<>();
+        for (int i = 0; i < 150; i++) {
+            requests.add(Tuple.tuple("/index/_bulk", bulkRequest));
+        }
+
+        HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
+        TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());
+
+        try (ReactorHttpClient nettyHttpClient = ReactorHttpClient.create(false)) {
+            final Collection singleResponse = nettyHttpClient.post(transportAddress.address(), requests.subList(0, 1));
+            try {
+                assertThat(singleResponse, hasSize(1));
+                assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
+
+                final Collection multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
+                try {
+                    assertThat(multipleResponses, hasSize(requests.size()));
+                    assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.TOO_MANY_REQUESTS);
+                } finally {
+                    multipleResponses.forEach(ReferenceCounted::release);
+                }
+            } finally {
+                singleResponse.forEach(ReferenceCounted::release);
+            }
+        }
+    }
+
+    public void testDoesNotLimitExcludedRequests() throws Exception {
+        ensureGreen();
+
+        List> requestUris = new ArrayList<>();
+        for (int i = 0; i < 1500; i++) {
+            requestUris.add(Tuple.tuple("/_cluster/settings", "{ \"transient\": {\"search.default_search_timeout\": \"40s\" } }"));
+        }
+
+        HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
+        TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());
+
+        try (ReactorHttpClient nettyHttpClient = ReactorHttpClient.create(false)) {
+            final Collection responses = nettyHttpClient.put(transportAddress.address(), requestUris);
+            try {
+                assertThat(responses, hasSize(requestUris.size()));
+                assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
+        }
+    }
+
+    private void assertAtLeastOnceExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) {
+        long countExpectedStatus = responses.stream().filter(r -> r.status().equals(expectedStatus)).count();
+        assertThat("Expected at least one request with status [" + expectedStatus + "]", countExpectedStatus, greaterThan(0L));
+    }
+
+    private void assertAllInExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) {
+        long countUnexpectedStatus = responses.stream().filter(r -> r.status().equals(expectedStatus) == false).count();
+        assertThat(
+            "Expected all requests with status [" + expectedStatus + "] but [" + countUnexpectedStatus + "] requests had a different one",
+            countUnexpectedStatus,
+            equalTo(0L)
+        );
+    }
+
+}
diff --git a/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4PipeliningIT.java b/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4PipeliningIT.java
new file mode 100644
index 0000000000000..c0e43de06f6ff
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/internalClusterTest/java/org/opensearch/http/reactor/netty4/ReactorNetty4PipeliningIT.java
@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.OpenSearchReactorNetty4IntegTestCase;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
+import org.opensearch.test.OpenSearchIntegTestCase.Scope;
+
+import java.util.Collection;
+import java.util.Locale;
+
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
+
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
+public class ReactorNetty4PipeliningIT extends OpenSearchReactorNetty4IntegTestCase {
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    public void testThatNettyHttpServerSupportsPipelining() throws Exception {
+        String[] requests = new String[] { "/", "/_nodes/stats", "/", "/_cluster/state", "/" };
+
+        HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
+        TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses();
+        TransportAddress transportAddress = randomFrom(boundAddresses);
+
+        try (ReactorHttpClient client = ReactorHttpClient.create()) {
+            Collection responses = client.get(transportAddress.address(), true, requests);
+            try {
+                assertThat(responses, hasSize(5));
+
+                Collection opaqueIds = ReactorHttpClient.returnOpaqueIds(responses);
+                assertOpaqueIdsInOrder(opaqueIds);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
+        }
+    }
+
+    private void assertOpaqueIdsInOrder(Collection opaqueIds) {
+        // check if opaque ids are monotonically increasing
+        int i = 0;
+        String msg = String.format(Locale.ROOT, "Expected list of opaque ids to be monotonically increasing, got [%s]", opaqueIds);
+        for (String opaqueId : opaqueIds) {
+            assertThat(msg, opaqueId, is(String.valueOf(i++)));
+        }
+    }
+
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/HttpConversionUtil.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/HttpConversionUtil.java
new file mode 100644
index 0000000000000..bd75227dabd08
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/HttpConversionUtil.java
@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.rest.RestRequest;
+
+import io.netty.handler.codec.http.HttpMethod;
+
+final class HttpConversionUtil {
+    private HttpConversionUtil() {}
+
+    /**
+     * Converts {@link HttpMethod} to {@link RestRequest.Method}
+     * @param method {@link HttpMethod} method
+     * @return corresponding {@link RestRequest.Method}
+     * @throws IllegalArgumentException if HTTP method is not supported
+     */
+    public static RestRequest.Method convertMethod(HttpMethod method) {
+        if (method == HttpMethod.GET) {
+            return RestRequest.Method.GET;
+        } else if (method == HttpMethod.POST) {
+            return RestRequest.Method.POST;
+        } else if (method == HttpMethod.PUT) {
+            return RestRequest.Method.PUT;
+        } else if (method == HttpMethod.DELETE) {
+            return RestRequest.Method.DELETE;
+        } else if (method == HttpMethod.HEAD) {
+            return RestRequest.Method.HEAD;
+        } else if (method == HttpMethod.OPTIONS) {
+            return RestRequest.Method.OPTIONS;
+        } else if (method == HttpMethod.PATCH) {
+            return RestRequest.Method.PATCH;
+        } else if (method == HttpMethod.TRACE) {
+            return RestRequest.Method.TRACE;
+        } else if (method == HttpMethod.CONNECT) {
+            return RestRequest.Method.CONNECT;
+        } else {
+            throw new IllegalArgumentException("Unexpected http method: " + method);
+        }
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingHttpChannel.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingHttpChannel.java
new file mode 100644
index 0000000000000..98b359319ff1b
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingHttpChannel.java
@@ -0,0 +1,76 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.common.concurrent.CompletableContext;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.http.HttpChannel;
+import org.opensearch.http.HttpResponse;
+import org.opensearch.transport.reactor.netty4.Netty4Utils;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
+import reactor.core.publisher.FluxSink;
+import reactor.netty.http.server.HttpServerRequest;
+import reactor.netty.http.server.HttpServerResponse;
+
+class NonStreamingHttpChannel implements HttpChannel {
+    private final HttpServerRequest request;
+    private final HttpServerResponse response;
+    private final CompletableContext closeContext = new CompletableContext<>();
+    private final FluxSink emitter;
+
+    NonStreamingHttpChannel(HttpServerRequest request, HttpServerResponse response, FluxSink emitter) {
+        this.request = request;
+        this.response = response;
+        this.emitter = emitter;
+        this.request.withConnection(connection -> Netty4Utils.addListener(connection.channel().closeFuture(), closeContext));
+    }
+
+    @Override
+    public boolean isOpen() {
+        final AtomicBoolean isOpen = new AtomicBoolean();
+        request.withConnection(connection -> isOpen.set(connection.channel().isOpen()));
+        return isOpen.get();
+    }
+
+    @Override
+    public void close() {
+        request.withConnection(connection -> connection.channel().close());
+    }
+
+    @Override
+    public void addCloseListener(ActionListener listener) {
+        closeContext.addListener(ActionListener.toBiConsumer(listener));
+    }
+
+    @Override
+    public void sendResponse(HttpResponse response, ActionListener listener) {
+        emitter.next(createResponse(response));
+        listener.onResponse(null);
+        emitter.complete();
+    }
+
+    @Override
+    public InetSocketAddress getRemoteAddress() {
+        return (InetSocketAddress) response.remoteAddress();
+    }
+
+    @Override
+    public InetSocketAddress getLocalAddress() {
+        return (InetSocketAddress) response.hostAddress();
+    }
+
+    FullHttpResponse createResponse(HttpResponse response) {
+        return (FullHttpResponse) response;
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingRequestConsumer.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingRequestConsumer.java
new file mode 100644
index 0000000000000..d43e23e800e65
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/NonStreamingRequestConsumer.java
@@ -0,0 +1,104 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.http.AbstractHttpServerTransport;
+import org.opensearch.http.HttpRequest;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+import io.netty.buffer.CompositeByteBuf;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.LastHttpContent;
+import org.reactivestreams.Publisher;
+import org.reactivestreams.Subscriber;
+import reactor.core.Disposable;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.FluxSink;
+import reactor.netty.http.server.HttpServerRequest;
+import reactor.netty.http.server.HttpServerResponse;
+
+class NonStreamingRequestConsumer implements Consumer, Publisher, Disposable {
+    private final HttpServerRequest request;
+    private final HttpServerResponse response;
+    private final CompositeByteBuf content;
+    private final Publisher publisher;
+    private final AbstractHttpServerTransport transport;
+    private final AtomicBoolean disposed = new AtomicBoolean(false);
+    private volatile FluxSink emitter;
+
+    NonStreamingRequestConsumer(
+        AbstractHttpServerTransport transport,
+        HttpServerRequest request,
+        HttpServerResponse response,
+        int maxCompositeBufferComponents
+    ) {
+        this.transport = transport;
+        this.request = request;
+        this.response = response;
+        this.content = response.alloc().compositeBuffer(maxCompositeBufferComponents);
+        this.publisher = Flux.create(emitter -> register(emitter));
+    }
+
+    private void register(FluxSink emitter) {
+        this.emitter = emitter.onDispose(this).onCancel(this);
+    }
+
+    @Override
+    public void accept(T message) {
+        try {
+            if (message instanceof LastHttpContent) {
+                process(message, emitter);
+            } else if (message instanceof HttpContent) {
+                process(message, emitter);
+            }
+        } catch (Throwable ex) {
+            emitter.error(ex);
+        }
+    }
+
+    public void process(HttpContent in, FluxSink emitter) {
+        // Consume request body in full before dispatching it
+        content.addComponent(true, in.content().retain());
+
+        if (in instanceof LastHttpContent) {
+            final NonStreamingHttpChannel channel = new NonStreamingHttpChannel(request, response, emitter);
+            final HttpRequest r = createRequest(request, content);
+
+            try {
+                transport.incomingRequest(r, channel);
+            } catch (Exception ex) {
+                emitter.error(ex);
+                transport.onException(channel, ex);
+            } finally {
+                r.release();
+                if (disposed.compareAndSet(false, true)) {
+                    this.content.release();
+                }
+            }
+        }
+    }
+
+    HttpRequest createRequest(HttpServerRequest request, CompositeByteBuf content) {
+        return new ReactorNetty4HttpRequest(request, content.retain());
+    }
+
+    @Override
+    public void subscribe(Subscriber s) {
+        publisher.subscribe(s);
+    }
+
+    @Override
+    public void dispose() {
+        if (disposed.compareAndSet(false, true)) {
+            this.content.release();
+        }
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequest.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequest.java
new file mode 100644
index 0000000000000..4406c555a5b04
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpRequest.java
@@ -0,0 +1,272 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.http.HttpRequest;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.transport.reactor.netty4.Netty4Utils;
+
+import java.util.AbstractMap;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.handler.codec.http.DefaultHttpHeaders;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.cookie.Cookie;
+import io.netty.handler.codec.http.cookie.ServerCookieDecoder;
+import io.netty.handler.codec.http.cookie.ServerCookieEncoder;
+import reactor.netty.http.server.HttpServerRequest;
+
+class ReactorNetty4HttpRequest implements HttpRequest {
+    private final String protocol;
+    private final HttpMethod method;
+    private final String uri;
+    private final ByteBuf content;
+    private final HttpHeadersMap headers;
+    private final AtomicBoolean released;
+    private final Exception inboundException;
+    private final boolean pooled;
+
+    ReactorNetty4HttpRequest(HttpServerRequest request, ByteBuf content) {
+        this(request, new HttpHeadersMap(request.requestHeaders()), new AtomicBoolean(false), true, content);
+    }
+
+    ReactorNetty4HttpRequest(HttpServerRequest request, ByteBuf content, Exception inboundException) {
+        this(
+            request.protocol(),
+            request.method(),
+            request.uri(),
+            new HttpHeadersMap(request.requestHeaders()),
+            new AtomicBoolean(false),
+            true,
+            content,
+            inboundException
+        );
+    }
+
+    private ReactorNetty4HttpRequest(
+        HttpServerRequest request,
+        HttpHeadersMap headers,
+        AtomicBoolean released,
+        boolean pooled,
+        ByteBuf content
+    ) {
+        this(request.protocol(), request.method(), request.uri(), headers, released, pooled, content, null);
+    }
+
+    private ReactorNetty4HttpRequest(
+        String protocol,
+        HttpMethod method,
+        String uri,
+        HttpHeadersMap headers,
+        AtomicBoolean released,
+        boolean pooled,
+        ByteBuf content,
+        Exception inboundException
+    ) {
+
+        this.protocol = protocol;
+        this.method = method;
+        this.uri = uri;
+        this.headers = headers;
+        this.content = content;
+        this.pooled = pooled;
+        this.released = released;
+        this.inboundException = inboundException;
+    }
+
+    @Override
+    public RestRequest.Method method() {
+        return HttpConversionUtil.convertMethod(method);
+    }
+
+    @Override
+    public String uri() {
+        return uri;
+    }
+
+    @Override
+    public BytesReference content() {
+        assert released.get() == false;
+        return Netty4Utils.toBytesReference(content);
+    }
+
+    @Override
+    public void release() {
+        if (pooled && released.compareAndSet(false, true)) {
+            content.release();
+        }
+    }
+
+    @Override
+    public HttpRequest releaseAndCopy() {
+        assert released.get() == false;
+        if (pooled == false) {
+            return this;
+        }
+        try {
+            final ByteBuf copiedContent = Unpooled.copiedBuffer(content);
+            return new ReactorNetty4HttpRequest(protocol, method, uri, headers, new AtomicBoolean(false), false, copiedContent, null);
+        } finally {
+            release();
+        }
+    }
+
+    @Override
+    public final Map> getHeaders() {
+        return headers;
+    }
+
+    @Override
+    public List strictCookies() {
+        String cookieString = headers.httpHeaders.get(HttpHeaderNames.COOKIE);
+        if (cookieString != null) {
+            Set cookies = ServerCookieDecoder.STRICT.decode(cookieString);
+            if (!cookies.isEmpty()) {
+                return ServerCookieEncoder.STRICT.encode(cookies);
+            }
+        }
+        return Collections.emptyList();
+    }
+
+    @Override
+    public HttpVersion protocolVersion() {
+        if (protocol.equals(io.netty.handler.codec.http.HttpVersion.HTTP_1_0.toString())) {
+            return HttpRequest.HttpVersion.HTTP_1_0;
+        } else if (protocol.equals(io.netty.handler.codec.http.HttpVersion.HTTP_1_1.toString())) {
+            return HttpRequest.HttpVersion.HTTP_1_1;
+        } else {
+            throw new IllegalArgumentException("Unexpected http protocol version: " + protocol);
+        }
+    }
+
+    @Override
+    public HttpRequest removeHeader(String header) {
+        HttpHeaders headersWithoutContentTypeHeader = new DefaultHttpHeaders();
+        headersWithoutContentTypeHeader.add(headers.httpHeaders);
+        headersWithoutContentTypeHeader.remove(header);
+
+        return new ReactorNetty4HttpRequest(
+            protocol,
+            method,
+            uri,
+            new HttpHeadersMap(headersWithoutContentTypeHeader),
+            released,
+            pooled,
+            content,
+            null
+        );
+    }
+
+    @Override
+    public ReactorNetty4HttpResponse createResponse(RestStatus status, BytesReference content) {
+        return new ReactorNetty4HttpResponse(
+            headers.httpHeaders,
+            io.netty.handler.codec.http.HttpVersion.valueOf(protocol),
+            status,
+            content
+        );
+    }
+
+    @Override
+    public Exception getInboundException() {
+        return inboundException;
+    }
+
+    /**
+     * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications
+     * and due to the underlying implementation, it performs case insensitive lookups of key to values.
+     *
+     * It is important to note that this implementation does have some downsides in that each invocation of the
+     * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a
+     * view of the underlying values.
+     */
+    private static class HttpHeadersMap implements Map> {
+
+        private final HttpHeaders httpHeaders;
+
+        private HttpHeadersMap(HttpHeaders httpHeaders) {
+            this.httpHeaders = httpHeaders;
+        }
+
+        @Override
+        public int size() {
+            return httpHeaders.size();
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return httpHeaders.isEmpty();
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            return key instanceof String && httpHeaders.contains((String) key);
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            return value instanceof List && httpHeaders.names().stream().map(httpHeaders::getAll).anyMatch(value::equals);
+        }
+
+        @Override
+        public List get(Object key) {
+            return key instanceof String ? httpHeaders.getAll((String) key) : null;
+        }
+
+        @Override
+        public List put(String key, List value) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public List remove(Object key) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void putAll(Map> m) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void clear() {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public Set keySet() {
+            return httpHeaders.names();
+        }
+
+        @Override
+        public Collection> values() {
+            return httpHeaders.names().stream().map(k -> Collections.unmodifiableList(httpHeaders.getAll(k))).collect(Collectors.toList());
+        }
+
+        @Override
+        public Set>> entrySet() {
+            return httpHeaders.names()
+                .stream()
+                .map(k -> new AbstractMap.SimpleImmutableEntry<>(k, httpHeaders.getAll(k)))
+                .collect(Collectors.toSet());
+        }
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpResponse.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpResponse.java
new file mode 100644
index 0000000000000..c45ad54b668a3
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpResponse.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.http.HttpResponse;
+import org.opensearch.transport.reactor.netty4.Netty4Utils;
+
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpVersion;
+
+class ReactorNetty4HttpResponse extends DefaultFullHttpResponse implements HttpResponse {
+    private final HttpHeaders requestHeaders;
+
+    ReactorNetty4HttpResponse(HttpHeaders requestHeaders, HttpVersion version, RestStatus status, BytesReference content) {
+        super(version, HttpResponseStatus.valueOf(status.getStatus()), Netty4Utils.toByteBuf(content));
+        this.requestHeaders = requestHeaders;
+    }
+
+    @Override
+    public void addHeader(String name, String value) {
+        headers().add(name, value);
+    }
+
+    @Override
+    public boolean containsHeader(String name) {
+        return headers().contains(name);
+    }
+
+    public HttpHeaders requestHeaders() {
+        return requestHeaders;
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerChannel.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerChannel.java
new file mode 100644
index 0000000000000..84360bf028ba9
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerChannel.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.common.concurrent.CompletableContext;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.http.HttpServerChannel;
+import org.opensearch.transport.reactor.netty4.Netty4Utils;
+
+import java.net.InetSocketAddress;
+
+import io.netty.channel.Channel;
+
+class ReactorNetty4HttpServerChannel implements HttpServerChannel {
+    private final Channel channel;
+    private final CompletableContext closeContext = new CompletableContext<>();
+
+    ReactorNetty4HttpServerChannel(Channel channel) {
+        this.channel = channel;
+        Netty4Utils.addListener(this.channel.closeFuture(), closeContext);
+    }
+
+    @Override
+    public InetSocketAddress getLocalAddress() {
+        return (InetSocketAddress) channel.localAddress();
+    }
+
+    @Override
+    public void addCloseListener(ActionListener listener) {
+        closeContext.addListener(ActionListener.toBiConsumer(listener));
+    }
+
+    @Override
+    public boolean isOpen() {
+        return channel.isOpen();
+    }
+
+    @Override
+    public void close() {
+        channel.close();
+    }
+
+    @Override
+    public String toString() {
+        return "ReactorNetty4HttpChannel{localAddress=" + getLocalAddress() + "}";
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java
new file mode 100644
index 0000000000000..d4a5a9ad83af6
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java
@@ -0,0 +1,313 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.common.network.NetworkService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Setting.Property;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
+import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.common.util.net.NetUtils;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.http.AbstractHttpServerTransport;
+import org.opensearch.http.HttpChannel;
+import org.opensearch.http.HttpReadTimeoutException;
+import org.opensearch.http.HttpServerChannel;
+import org.opensearch.telemetry.tracing.Tracer;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.reactor.SharedGroupFactory;
+import org.opensearch.transport.reactor.netty4.Netty4Utils;
+
+import java.net.InetSocketAddress;
+import java.net.SocketOption;
+import java.time.Duration;
+
+import io.netty.channel.ChannelOption;
+import io.netty.channel.socket.nio.NioChannelOption;
+import io.netty.handler.codec.http.DefaultLastHttpContent;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.timeout.ReadTimeoutException;
+import org.reactivestreams.Publisher;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Scheduler;
+import reactor.core.scheduler.Schedulers;
+import reactor.netty.DisposableServer;
+import reactor.netty.http.HttpProtocol;
+import reactor.netty.http.server.HttpServer;
+import reactor.netty.http.server.HttpServerRequest;
+import reactor.netty.http.server.HttpServerResponse;
+
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_CONNECT_TIMEOUT;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_KEEP_ALIVE;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_KEEP_COUNT;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_KEEP_IDLE;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_KEEP_INTERVAL;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_NO_DELAY;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_REUSE_ADDRESS;
+import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
+
+/**
+ * The HTTP transport implementations based on Reactor Netty (see please {@link HttpServer}).
+ */
+public class ReactorNetty4HttpServerTransport extends AbstractHttpServerTransport {
+    private static final String SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = "http.netty.max_composite_buffer_components";
+    private static final ByteSizeValue MTU = new ByteSizeValue(Long.parseLong(System.getProperty("opensearch.net.mtu", "1500")));
+
+    /**
+     * The number of Reactor Netty HTTP workers
+     */
+    public static final Setting SETTING_HTTP_WORKER_COUNT = Setting.intSetting("http.netty.worker_count", 0, Property.NodeScope);
+
+    /**
+     * The maximum number of composite components for request accumulation
+     */
+    public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = new Setting<>(
+        SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
+        (s) -> {
+            ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(s);
+            /*
+             * Netty accumulates buffers containing data from all incoming network packets that make up one HTTP request in an instance of
+             * io.netty.buffer.CompositeByteBuf (think of it as a buffer of buffers). Once its capacity is reached, the buffer will iterate
+             * over its individual entries and put them into larger buffers (see io.netty.buffer.CompositeByteBuf#consolidateIfNeeded()
+             * for implementation details). We want to to resize that buffer because this leads to additional garbage on the heap and also
+             * increases the application's native memory footprint (as direct byte buffers hold their contents off-heap).
+             *
+             * With this setting we control the CompositeByteBuf's capacity (which is by default 1024, see
+             * io.netty.handler.codec.MessageAggregator#DEFAULT_MAX_COMPOSITEBUFFER_COMPONENTS). To determine a proper default capacity for
+             * that buffer, we need to consider that the upper bound for the size of HTTP requests is determined by `maxContentLength`. The
+             * number of buffers that are needed depend on how often Netty reads network packets which depends on the network type (MTU).
+             * We assume here that OpenSearch receives HTTP requests via an Ethernet connection which has a MTU of 1500 bytes.
+             *
+             * Note that we are *not* pre-allocating any memory based on this setting but rather determine the CompositeByteBuf's capacity.
+             * The tradeoff is between less (but larger) buffers that are contained in the CompositeByteBuf and more (but smaller) buffers.
+             * With the default max content length of 100MB and a MTU of 1500 bytes we would allow 69905 entries.
+             */
+            long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MTU.getBytes()));
+            // clamp value to the allowed range
+            long maxBufferComponents = Math.max(2, Math.min(maxBufferComponentsEstimate, Integer.MAX_VALUE));
+            return String.valueOf(maxBufferComponents);
+            // Netty's CompositeByteBuf implementation does not allow less than two components.
+        },
+        s -> Setting.parseInt(s, 2, Integer.MAX_VALUE, SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS),
+        Property.NodeScope
+    );
+
+    private final SharedGroupFactory sharedGroupFactory;
+    private final int readTimeoutMillis;
+    private final int connectTimeoutMillis;
+    private final int maxCompositeBufferComponents;
+    private final ByteSizeValue maxInitialLineLength;
+    private final ByteSizeValue maxHeaderSize;
+    private final ByteSizeValue maxChunkSize;
+    private volatile SharedGroupFactory.SharedGroup sharedGroup;
+    private volatile DisposableServer disposableServer;
+    private volatile Scheduler scheduler;
+
+    /**
+     * Creates new HTTP transport implementations based on Reactor Netty (see please {@link HttpServer}).
+     * @param settings settings
+     * @param networkService network service
+     * @param bigArrays big array allocator
+     * @param threadPool thread pool instance
+     * @param xContentRegistry XContent registry instance
+     * @param dispatcher dispatcher instance
+     * @param clusterSettings cluster settings
+     * @param sharedGroupFactory shared group factory
+     * @param tracer tracer instance
+     */
+    public ReactorNetty4HttpServerTransport(
+        Settings settings,
+        NetworkService networkService,
+        BigArrays bigArrays,
+        ThreadPool threadPool,
+        NamedXContentRegistry xContentRegistry,
+        Dispatcher dispatcher,
+        ClusterSettings clusterSettings,
+        SharedGroupFactory sharedGroupFactory,
+        Tracer tracer
+    ) {
+        super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, tracer);
+        Netty4Utils.setAvailableProcessors(OpenSearchExecutors.NODE_PROCESSORS_SETTING.get(settings));
+        this.readTimeoutMillis = Math.toIntExact(SETTING_HTTP_READ_TIMEOUT.get(settings).getMillis());
+        this.connectTimeoutMillis = Math.toIntExact(SETTING_HTTP_CONNECT_TIMEOUT.get(settings).getMillis());
+        this.sharedGroupFactory = sharedGroupFactory;
+        this.maxCompositeBufferComponents = SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
+        this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings);
+        this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings);
+        this.maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings);
+    }
+
+    /**
+     * Binds the transport engine to the socket address
+     * @param socketAddress socket address to bind to
+     */
+    @Override
+    protected HttpServerChannel bind(InetSocketAddress socketAddress) throws Exception {
+        final HttpServer server = configureChannelOptions(
+            HttpServer.create()
+                .httpFormDecoder(builder -> builder.scheduler(scheduler))
+                .idleTimeout(Duration.ofMillis(connectTimeoutMillis))
+                .readTimeout(Duration.ofMillis(readTimeoutMillis))
+                .runOn(sharedGroup.getLowLevelGroup())
+                .bindAddress(() -> socketAddress)
+                .compress(true)
+                .httpRequestDecoder(
+                    spec -> spec.maxChunkSize(maxChunkSize.bytesAsInt())
+                        .maxHeaderSize(maxHeaderSize.bytesAsInt())
+                        .maxInitialLineLength(maxInitialLineLength.bytesAsInt())
+                )
+                .protocol(HttpProtocol.HTTP11, HttpProtocol.H2C)
+                .handle((req, res) -> incomingRequest(req, res))
+        );
+
+        disposableServer = server.bindNow();
+        return new ReactorNetty4HttpServerChannel(disposableServer.channel());
+    }
+
+    private HttpServer configureChannelOptions(final HttpServer server1) {
+        HttpServer configured = server1.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings))
+            .childOption(ChannelOption.SO_KEEPALIVE, SETTING_HTTP_TCP_KEEP_ALIVE.get(settings));
+
+        if (SETTING_HTTP_TCP_KEEP_ALIVE.get(settings)) {
+            // Netty logs a warning if it can't set the option, so try this only on supported platforms
+            if (IOUtils.LINUX || IOUtils.MAC_OS_X) {
+                if (SETTING_HTTP_TCP_KEEP_IDLE.get(settings) >= 0) {
+                    final SocketOption keepIdleOption = NetUtils.getTcpKeepIdleSocketOptionOrNull();
+                    if (keepIdleOption != null) {
+                        configured = configured.childOption(NioChannelOption.of(keepIdleOption), SETTING_HTTP_TCP_KEEP_IDLE.get(settings));
+                    }
+                }
+                if (SETTING_HTTP_TCP_KEEP_INTERVAL.get(settings) >= 0) {
+                    final SocketOption keepIntervalOption = NetUtils.getTcpKeepIntervalSocketOptionOrNull();
+                    if (keepIntervalOption != null) {
+                        configured = configured.childOption(
+                            NioChannelOption.of(keepIntervalOption),
+                            SETTING_HTTP_TCP_KEEP_INTERVAL.get(settings)
+                        );
+                    }
+                }
+                if (SETTING_HTTP_TCP_KEEP_COUNT.get(settings) >= 0) {
+                    final SocketOption keepCountOption = NetUtils.getTcpKeepCountSocketOptionOrNull();
+                    if (keepCountOption != null) {
+                        configured = configured.childOption(
+                            NioChannelOption.of(keepCountOption),
+                            SETTING_HTTP_TCP_KEEP_COUNT.get(settings)
+                        );
+                    }
+                }
+            }
+        }
+
+        final ByteSizeValue tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings);
+        if (tcpSendBufferSize.getBytes() > 0) {
+            configured = configured.childOption(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.getBytes()));
+        }
+
+        final ByteSizeValue tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings);
+        if (tcpReceiveBufferSize.getBytes() > 0) {
+            configured = configured.childOption(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.getBytes()));
+        }
+
+        final boolean reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings);
+        configured = configured.option(ChannelOption.SO_REUSEADDR, reuseAddress);
+        configured = configured.childOption(ChannelOption.SO_REUSEADDR, reuseAddress);
+
+        return configured;
+    }
+
+    /**
+     * Handles incoming Reactor Netty request
+     * @param request request instance
+     * @param response response instances
+     * @return response publisher
+     */
+    protected Publisher incomingRequest(HttpServerRequest request, HttpServerResponse response) {
+        final NonStreamingRequestConsumer consumer = new NonStreamingRequestConsumer<>(
+            this,
+            request,
+            response,
+            maxCompositeBufferComponents
+        );
+
+        request.receiveContent().switchIfEmpty(Mono.just(DefaultLastHttpContent.EMPTY_LAST_CONTENT)).subscribe(consumer);
+
+        return Mono.from(consumer).flatMap(hc -> {
+            final FullHttpResponse r = (FullHttpResponse) hc;
+            response.status(r.status());
+            response.trailerHeaders(c -> r.trailingHeaders().forEach(h -> c.add(h.getKey(), h.getValue())));
+            response.chunkedTransfer(false);
+            response.compression(true);
+            r.headers().forEach(h -> response.addHeader(h.getKey(), h.getValue()));
+            return Mono.from(response.sendObject(r.content()));
+        });
+    }
+
+    /**
+     * Called to tear down internal resources
+     */
+    @Override
+    protected void stopInternal() {
+        if (sharedGroup != null) {
+            sharedGroup.shutdown();
+            sharedGroup = null;
+        }
+
+        if (scheduler != null) {
+            scheduler.dispose();
+            scheduler = null;
+        }
+
+        if (disposableServer != null) {
+            disposableServer.disposeNow();
+            disposableServer = null;
+        }
+    }
+
+    /**
+     * Starts the transport
+     */
+    @Override
+    protected void doStart() {
+        boolean success = false;
+        try {
+            scheduler = Schedulers.newBoundedElastic(
+                Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE,
+                Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE,
+                "http-form-decoder"
+            );
+            sharedGroup = sharedGroupFactory.getHttpGroup();
+            bindServer();
+            success = true;
+        } finally {
+            if (success == false) {
+                doStop(); // otherwise we leak threads since we never moved to started
+            }
+        }
+    }
+
+    @Override
+    public void onException(HttpChannel channel, Exception cause) {
+        if (cause instanceof ReadTimeoutException) {
+            super.onException(channel, new HttpReadTimeoutException(readTimeoutMillis, cause));
+        } else {
+            super.onException(channel, cause);
+        }
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/package-info.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/package-info.java
new file mode 100644
index 0000000000000..b5ecb0b62f79d
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * The new HTTP transport implementations based on Reactor Netty.
+ */
+package org.opensearch.http.reactor.netty4;
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java
new file mode 100644
index 0000000000000..dc310c3793109
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java
@@ -0,0 +1,109 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.transport.reactor;
+
+import org.opensearch.common.SetOnce;
+import org.opensearch.common.network.NetworkService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.PageCacheRecycler;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.reactor.netty4.ReactorNetty4HttpServerTransport;
+import org.opensearch.plugins.NetworkPlugin;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.telemetry.tracing.Tracer;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+/**
+ * The experimental network plugin that introduces new transport implementations based on Reactor Netty.
+ */
+public class ReactorNetty4Plugin extends Plugin implements NetworkPlugin {
+    /**
+     * The name of new experimental HTTP transport implementations based on Reactor Netty.
+     */
+    public static final String REACTOR_NETTY_HTTP_TRANSPORT_NAME = "reactor-netty4";
+
+    private final SetOnce groupFactory = new SetOnce<>();
+
+    /**
+     * Default constructor
+     */
+    public ReactorNetty4Plugin() {}
+
+    /**
+     * Returns a list of additional {@link Setting} definitions for this plugin.
+     */
+    @Override
+    public List> getSettings() {
+        return Arrays.asList(/* no setting registered since we're picking the onces from Netty 4 transport */);
+    }
+
+    /**
+     * Returns a map of {@link HttpServerTransport} suppliers.
+     * See {@link org.opensearch.common.network.NetworkModule#HTTP_TYPE_SETTING} to configure a specific implementation.
+     * @param settings settings
+     * @param networkService network service
+     * @param bigArrays big array allocator
+     * @param pageCacheRecycler page cache recycler instance
+     * @param circuitBreakerService circuit breaker service instance
+     * @param threadPool thread pool instance
+     * @param xContentRegistry XContent registry instance
+     * @param dispatcher dispatcher instance
+     * @param clusterSettings cluster settings
+     * @param tracer tracer instance
+     */
+    @Override
+    public Map> getHttpTransports(
+        Settings settings,
+        ThreadPool threadPool,
+        BigArrays bigArrays,
+        PageCacheRecycler pageCacheRecycler,
+        CircuitBreakerService circuitBreakerService,
+        NamedXContentRegistry xContentRegistry,
+        NetworkService networkService,
+        HttpServerTransport.Dispatcher dispatcher,
+        ClusterSettings clusterSettings,
+        Tracer tracer
+    ) {
+        return Collections.singletonMap(
+            REACTOR_NETTY_HTTP_TRANSPORT_NAME,
+            () -> new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry,
+                dispatcher,
+                clusterSettings,
+                getSharedGroupFactory(settings),
+                tracer
+            )
+        );
+    }
+
+    private SharedGroupFactory getSharedGroupFactory(Settings settings) {
+        final SharedGroupFactory groupFactory = this.groupFactory.get();
+        if (groupFactory != null) {
+            assert groupFactory.getSettings().equals(settings) : "Different settings than originally provided";
+            return groupFactory;
+        } else {
+            this.groupFactory.set(new SharedGroupFactory(settings));
+            return this.groupFactory.get();
+        }
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/SharedGroupFactory.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/SharedGroupFactory.java
new file mode 100644
index 0000000000000..ab7de33c8e673
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/SharedGroupFactory.java
@@ -0,0 +1,164 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.transport.reactor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.AbstractRefCounted;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.reactor.netty4.ReactorNetty4HttpServerTransport;
+import org.opensearch.transport.TcpTransport;
+import org.opensearch.transport.reactor.netty4.ReactorNetty4Transport;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.util.concurrent.Future;
+
+import static org.opensearch.common.util.concurrent.OpenSearchExecutors.daemonThreadFactory;
+
+/**
+ * Creates and returns {@link io.netty.channel.EventLoopGroup} instances. It will return a shared group for
+ * both {@link #getHttpGroup()} and {@link #getTransportGroup()} if
+ * {@link org.opensearch.http.reactor.netty4.ReactorNetty4HttpServerTransport#SETTING_HTTP_WORKER_COUNT} is configured to be 0.
+ * If that setting is not 0, then it will return a different group in the {@link #getHttpGroup()} call.
+ */
+public final class SharedGroupFactory {
+
+    private static final Logger logger = LogManager.getLogger(SharedGroupFactory.class);
+
+    private final Settings settings;
+    private final int workerCount;
+    private final int httpWorkerCount;
+
+    private RefCountedGroup genericGroup;
+    private SharedGroup dedicatedHttpGroup;
+
+    /**
+     * Creates new shared group factory instance from settings
+     * @param settings settings
+     */
+    public SharedGroupFactory(Settings settings) {
+        this.settings = settings;
+        this.workerCount = ReactorNetty4Transport.SETTING_WORKER_COUNT.get(settings);
+        this.httpWorkerCount = ReactorNetty4HttpServerTransport.SETTING_HTTP_WORKER_COUNT.get(settings);
+    }
+
+    Settings getSettings() {
+        return settings;
+    }
+
+    /**
+     * Gets the number of configured transport workers
+     * @return the number of configured transport workers
+     */
+    public int getTransportWorkerCount() {
+        return workerCount;
+    }
+
+    /**
+     * Gets transport shared group
+     * @return transport shared group
+     */
+    public synchronized SharedGroup getTransportGroup() {
+        return getGenericGroup();
+    }
+
+    /**
+     * Gets HTTP transport shared group
+     * @return HTTP transport shared group
+     */
+    public synchronized SharedGroup getHttpGroup() {
+        if (httpWorkerCount == 0) {
+            return getGenericGroup();
+        } else {
+            if (dedicatedHttpGroup == null) {
+                NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(
+                    httpWorkerCount,
+                    daemonThreadFactory(settings, HttpServerTransport.HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)
+                );
+                dedicatedHttpGroup = new SharedGroup(new RefCountedGroup(eventLoopGroup));
+            }
+            return dedicatedHttpGroup;
+        }
+    }
+
+    private SharedGroup getGenericGroup() {
+        if (genericGroup == null) {
+            EventLoopGroup eventLoopGroup = new NioEventLoopGroup(
+                workerCount,
+                daemonThreadFactory(settings, TcpTransport.TRANSPORT_WORKER_THREAD_NAME_PREFIX)
+            );
+            this.genericGroup = new RefCountedGroup(eventLoopGroup);
+        } else {
+            genericGroup.incRef();
+        }
+        return new SharedGroup(genericGroup);
+    }
+
+    private static class RefCountedGroup extends AbstractRefCounted {
+
+        public static final String NAME = "ref-counted-event-loop-group";
+        private final EventLoopGroup eventLoopGroup;
+
+        private RefCountedGroup(EventLoopGroup eventLoopGroup) {
+            super(NAME);
+            this.eventLoopGroup = eventLoopGroup;
+        }
+
+        @Override
+        protected void closeInternal() {
+            Future shutdownFuture = eventLoopGroup.shutdownGracefully(0, 5, TimeUnit.SECONDS);
+            shutdownFuture.awaitUninterruptibly();
+            if (shutdownFuture.isSuccess() == false) {
+                logger.warn("Error closing netty event loop group", shutdownFuture.cause());
+            }
+        }
+    }
+
+    /**
+     * Wraps the {@link RefCountedGroup}. Calls {@link RefCountedGroup#decRef()} on close. After close,
+     * this wrapped instance can no longer be used.
+     */
+    public static class SharedGroup {
+
+        private final RefCountedGroup refCountedGroup;
+
+        private final AtomicBoolean isOpen = new AtomicBoolean(true);
+
+        private SharedGroup(RefCountedGroup refCountedGroup) {
+            this.refCountedGroup = refCountedGroup;
+        }
+
+        /**
+         * Gets Netty's {@link EventLoopGroup} instance
+         * @return Netty's {@link EventLoopGroup} instance
+         */
+        public EventLoopGroup getLowLevelGroup() {
+            return refCountedGroup.eventLoopGroup;
+        }
+
+        /**
+         * Decreases the reference to underlying  {@link EventLoopGroup} instance
+         */
+        public void shutdown() {
+            if (isOpen.compareAndSet(true, false)) {
+                refCountedGroup.decRef();
+            }
+        }
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/Netty4Utils.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/Netty4Utils.java
new file mode 100644
index 0000000000000..8ec432b7dd5cd
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/Netty4Utils.java
@@ -0,0 +1,142 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.transport.reactor.netty4;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.opensearch.ExceptionsHelper;
+import org.opensearch.common.Booleans;
+import org.opensearch.common.concurrent.CompletableContext;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.CompositeByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelFuture;
+import io.netty.util.NettyRuntime;
+
+/**
+ * Shameless copy of Netty4Utils from transport-netty4 module
+ */
+public final class Netty4Utils {
+    private static final AtomicBoolean isAvailableProcessorsSet = new AtomicBoolean();
+
+    /**
+     * Utility class
+     */
+    private Netty4Utils() {}
+
+    /**
+     * Set the number of available processors that Netty uses for sizing various resources (e.g., thread pools).
+     *
+     * @param availableProcessors the number of available processors
+     * @throws IllegalStateException if available processors was set previously and the specified value does not match the already-set value
+     */
+    public static void setAvailableProcessors(final int availableProcessors) {
+        // we set this to false in tests to avoid tests that randomly set processors from stepping on each other
+        final boolean set = Booleans.parseBoolean(System.getProperty("opensearch.set.netty.runtime.available.processors", "true"));
+        if (!set) {
+            return;
+        }
+
+        /*
+         * This can be invoked twice, once from Netty4Transport and another time from Netty4HttpServerTransport; however,
+         * Netty4Runtime#availableProcessors forbids settings the number of processors twice so we prevent double invocation here.
+         */
+        if (isAvailableProcessorsSet.compareAndSet(false, true)) {
+            NettyRuntime.setAvailableProcessors(availableProcessors);
+        } else if (availableProcessors != NettyRuntime.availableProcessors()) {
+            /*
+             * We have previously set the available processors yet either we are trying to set it to a different value now or there is a bug
+             * in Netty and our previous value did not take, bail.
+             */
+            final String message = String.format(
+                Locale.ROOT,
+                "available processors value [%d] did not match current value [%d]",
+                availableProcessors,
+                NettyRuntime.availableProcessors()
+            );
+            throw new IllegalStateException(message);
+        }
+    }
+
+    /**
+     * Turns the given BytesReference into a ByteBuf. Note: the returned ByteBuf will reference the internal
+     * pages of the BytesReference. Don't free the bytes of reference before the ByteBuf goes out of scope.
+     * @param reference reference to convert
+     */
+    public static ByteBuf toByteBuf(final BytesReference reference) {
+        if (reference.length() == 0) {
+            return Unpooled.EMPTY_BUFFER;
+        }
+        final BytesRefIterator iterator = reference.iterator();
+        // usually we have one, two, or three components from the header, the message, and a buffer
+        final List buffers = new ArrayList<>(3);
+        try {
+            BytesRef slice;
+            while ((slice = iterator.next()) != null) {
+                buffers.add(Unpooled.wrappedBuffer(slice.bytes, slice.offset, slice.length));
+            }
+
+            if (buffers.size() == 1) {
+                return buffers.get(0);
+            } else {
+                CompositeByteBuf composite = Unpooled.compositeBuffer(buffers.size());
+                composite.addComponents(true, buffers);
+                return composite;
+            }
+        } catch (IOException ex) {
+            throw new AssertionError("no IO happens here", ex);
+        }
+    }
+
+    /**
+     * Wraps the given ChannelBuffer with a BytesReference
+     * @param buffer buffer to convert
+     */
+    public static BytesReference toBytesReference(final ByteBuf buffer) {
+        final int readableBytes = buffer.readableBytes();
+        if (readableBytes == 0) {
+            return BytesArray.EMPTY;
+        } else if (buffer.hasArray()) {
+            return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), readableBytes);
+        } else {
+            final ByteBuffer[] byteBuffers = buffer.nioBuffers();
+            return BytesReference.fromByteBuffers(byteBuffers);
+        }
+    }
+
+    /**
+     * Add completion listener to ChannelFuture
+     * @param channelFuture ChannelFuture to add listener to
+     * @param context completion listener context
+     */
+    public static void addListener(ChannelFuture channelFuture, CompletableContext context) {
+        channelFuture.addListener(f -> {
+            if (f.isSuccess()) {
+                context.complete(null);
+            } else {
+                Throwable cause = f.cause();
+                if (cause instanceof Error) {
+                    ExceptionsHelper.maybeDieOnAnotherThread(cause);
+                    context.completeExceptionally(new Exception(cause));
+                } else {
+                    context.completeExceptionally((Exception) cause);
+                }
+            }
+        });
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/ReactorNetty4Transport.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/ReactorNetty4Transport.java
new file mode 100644
index 0000000000000..b3e92f58c540a
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/ReactorNetty4Transport.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.reactor.netty4;
+
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Setting.Property;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
+
+import reactor.netty.tcp.TcpServer;
+
+/**
+ * The transport implementations based on Reactor Netty (see please {@link TcpServer}).
+ */
+public class ReactorNetty4Transport {
+    /**
+     * The number of Netty workers
+     */
+    public static final Setting SETTING_WORKER_COUNT = new Setting<>(
+        "transport.netty.worker_count",
+        (s) -> Integer.toString(OpenSearchExecutors.allocatedProcessors(s)),
+        (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"),
+        Property.NodeScope
+    );
+
+    /**
+     * Default constructor
+     */
+    public ReactorNetty4Transport() {}
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/package-info.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/package-info.java
new file mode 100644
index 0000000000000..921bca104c6fe
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/netty4/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * The new transport implementations based on Reactor Netty.
+ */
+package org.opensearch.transport.reactor.netty4;
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/package-info.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/package-info.java
new file mode 100644
index 0000000000000..2f36ebb7f11f8
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * The experimental network plugin that introduces new transport implementations based on Reactor Netty.
+ */
+package org.opensearch.transport.reactor;
diff --git a/plugins/transport-reactor-netty4/src/main/plugin-metadata/plugin-security.policy b/plugins/transport-reactor-netty4/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 0000000000000..4f2dcde995338
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+grant codeBase "${codebase.netty-common}" {
+   // for reading the system-wide configuration for the backlog of established sockets
+   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+
+   // netty makes and accepts socket connections
+   permission java.net.SocketPermission "*", "accept,connect";
+
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
+};
+
+grant codeBase "${codebase.netty-transport}" {
+   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
+   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
+   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
+};
diff --git a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java
new file mode 100644
index 0000000000000..443ecd0f40ead
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java
@@ -0,0 +1,208 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.tasks.Task;
+
+import java.io.Closeable;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.EmptyHttpHeaders;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.handler.codec.http2.HttpConversionUtil;
+import io.netty.resolver.DefaultAddressResolverGroup;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import reactor.core.publisher.ParallelFlux;
+import reactor.netty.http.client.HttpClient;
+
+import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+
+/**
+ * Tiny helper to send http requests over netty.
+ */
+class ReactorHttpClient implements Closeable {
+    private final boolean compression;
+
+    static Collection returnHttpResponseBodies(Collection responses) {
+        List list = new ArrayList<>(responses.size());
+        for (FullHttpResponse response : responses) {
+            list.add(response.content().toString(StandardCharsets.UTF_8));
+        }
+        return list;
+    }
+
+    static Collection returnOpaqueIds(Collection responses) {
+        List list = new ArrayList<>(responses.size());
+        for (HttpResponse response : responses) {
+            list.add(response.headers().get(Task.X_OPAQUE_ID));
+        }
+        return list;
+    }
+
+    ReactorHttpClient(boolean compression) {
+        this.compression = compression;
+    }
+
+    static ReactorHttpClient create() {
+        return create(true);
+    }
+
+    static ReactorHttpClient create(boolean compression) {
+        return new ReactorHttpClient(compression);
+    }
+
+    public List get(InetSocketAddress remoteAddress, String... uris) throws InterruptedException {
+        return get(remoteAddress, false, uris);
+    }
+
+    public List get(InetSocketAddress remoteAddress, boolean ordered, String... uris) throws InterruptedException {
+        final List requests = new ArrayList<>(uris.length);
+
+        for (int i = 0; i < uris.length; i++) {
+            final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uris[i]);
+            httpRequest.headers().add(HOST, "localhost");
+            httpRequest.headers().add("X-Opaque-ID", String.valueOf(i));
+            httpRequest.headers().add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), "http");
+            requests.add(httpRequest);
+        }
+
+        return sendRequests(remoteAddress, requests, ordered);
+    }
+
+    public final Collection post(InetSocketAddress remoteAddress, List> urisAndBodies)
+        throws InterruptedException {
+        return processRequestsWithBody(HttpMethod.POST, remoteAddress, urisAndBodies);
+    }
+
+    public final FullHttpResponse send(InetSocketAddress remoteAddress, FullHttpRequest httpRequest) throws InterruptedException {
+        final List responses = sendRequests(remoteAddress, Collections.singleton(httpRequest), false);
+        assert responses.size() == 1 : "expected 1 and only 1 http response";
+        return responses.get(0);
+    }
+
+    public final FullHttpResponse send(InetSocketAddress remoteAddress, FullHttpRequest httpRequest, HttpContent content)
+        throws InterruptedException {
+        final List responses = sendRequests(
+            remoteAddress,
+            Collections.singleton(
+                new DefaultFullHttpRequest(
+                    httpRequest.protocolVersion(),
+                    httpRequest.method(),
+                    httpRequest.uri(),
+                    content.content(),
+                    httpRequest.headers(),
+                    httpRequest.trailingHeaders()
+                )
+            ),
+            false
+        );
+        assert responses.size() == 1 : "expected 1 and only 1 http response";
+        return responses.get(0);
+    }
+
+    public final Collection put(InetSocketAddress remoteAddress, List> urisAndBodies)
+        throws InterruptedException {
+        return processRequestsWithBody(HttpMethod.PUT, remoteAddress, urisAndBodies);
+    }
+
+    private List processRequestsWithBody(
+        HttpMethod method,
+        InetSocketAddress remoteAddress,
+        List> urisAndBodies
+    ) throws InterruptedException {
+        List requests = new ArrayList<>(urisAndBodies.size());
+        for (int i = 0; i < urisAndBodies.size(); ++i) {
+            final Tuple uriAndBody = urisAndBodies.get(i);
+            ByteBuf content = Unpooled.copiedBuffer(uriAndBody.v2(), StandardCharsets.UTF_8);
+            FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content);
+            request.headers().add(HttpHeaderNames.HOST, "localhost");
+            request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes());
+            request.headers().add(HttpHeaderNames.CONTENT_TYPE, "application/json");
+            request.headers().add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), "http");
+            request.headers().add("X-Opaque-ID", String.valueOf(i));
+            requests.add(request);
+        }
+        return sendRequests(remoteAddress, requests, false);
+    }
+
+    private List sendRequests(
+        final InetSocketAddress remoteAddress,
+        final Collection requests,
+        boolean orderer
+    ) {
+        final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(1);
+        try {
+            final HttpClient client = HttpClient.newConnection()
+                .resolver(DefaultAddressResolverGroup.INSTANCE)
+                .runOn(eventLoopGroup)
+                .host(remoteAddress.getHostString())
+                .port(remoteAddress.getPort())
+                .compress(compression);
+
+            @SuppressWarnings("unchecked")
+            final Mono[] monos = requests.stream()
+                .map(
+                    request -> client.headers(h -> h.add(request.headers()))
+                        .baseUrl(request.getUri())
+                        .request(request.method())
+                        .send(Mono.fromSupplier(() -> request.content()))
+                        .responseSingle(
+                            (r, body) -> body.switchIfEmpty(Mono.just(Unpooled.EMPTY_BUFFER))
+                                .map(
+                                    b -> new DefaultFullHttpResponse(
+                                        r.version(),
+                                        r.status(),
+                                        b.retain(),
+                                        r.responseHeaders(),
+                                        EmptyHttpHeaders.INSTANCE
+                                    )
+                                )
+                        )
+                )
+                .toArray(Mono[]::new);
+
+            if (orderer == false) {
+                return ParallelFlux.from(monos).sequential().collectList().block();
+            } else {
+                return Flux.concat(monos).flatMapSequential(r -> Mono.just(r)).collectList().block();
+            }
+        } finally {
+            eventLoopGroup.shutdownGracefully().awaitUninterruptibly();
+        }
+    }
+
+    @Override
+    public void close() {
+
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4BadRequestTests.java b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4BadRequestTests.java
new file mode 100644
index 0000000000000..00ca378a4e46b
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4BadRequestTests.java
@@ -0,0 +1,122 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.common.network.NetworkService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.MockBigArrays;
+import org.opensearch.common.util.MockPageCacheRecycler;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.HttpTransportSettings;
+import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.RestChannel;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.telemetry.tracing.noop.NoopTracer;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.reactor.SharedGroupFactory;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+public class ReactorNetty4BadRequestTests extends OpenSearchTestCase {
+
+    private NetworkService networkService;
+    private MockBigArrays bigArrays;
+    private ThreadPool threadPool;
+
+    @Before
+    public void setup() throws Exception {
+        networkService = new NetworkService(Collections.emptyList());
+        bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
+        threadPool = new TestThreadPool("test");
+    }
+
+    @After
+    public void shutdown() throws Exception {
+        terminate(threadPool);
+    }
+
+    public void testBadParameterEncoding() throws Exception {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+                fail();
+            }
+
+            @Override
+            public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause) {
+                try {
+                    final Exception e = cause instanceof Exception ? (Exception) cause : new OpenSearchException(cause);
+                    channel.sendResponse(new BytesRestResponse(channel, RestStatus.BAD_REQUEST, e));
+                } catch (final IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        };
+
+        Settings settings = Settings.builder().put(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), getPortRange()).build();
+        try (
+            HttpServerTransport httpServerTransport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                new SharedGroupFactory(Settings.EMPTY),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            httpServerTransport.start();
+            final TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());
+
+            try (ReactorHttpClient nettyHttpClient = ReactorHttpClient.create()) {
+                final List responses = nettyHttpClient.get(transportAddress.address(), "/_cluster/settings?pretty=%");
+
+                try {
+                    assertThat(responses, hasSize(1));
+                    final FullHttpResponse response = responses.get(0);
+                    assertThat(response.status().code(), equalTo(400));
+                    final Collection responseBodies = ReactorHttpClient.returnHttpResponseBodies(responses);
+                    assertThat(responseBodies, hasSize(1));
+                    final String body = responseBodies.iterator().next();
+                    assertThat(body, containsString("\"type\":\"bad_parameter_exception\""));
+                    assertThat(
+                        body,
+                        containsString("\"reason\":\"java.lang.IllegalArgumentException: partial escape sequence at end of string: %/\"")
+                    );
+                } finally {
+                    responses.forEach(ReferenceCounted::release);
+                }
+            }
+        }
+    }
+
+}
diff --git a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransportTests.java b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransportTests.java
new file mode 100644
index 0000000000000..15a5b04c802a4
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransportTests.java
@@ -0,0 +1,579 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.http.reactor.netty4;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.common.network.NetworkAddress;
+import org.opensearch.common.network.NetworkService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.MockBigArrays;
+import org.opensearch.common.util.MockPageCacheRecycler;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.http.BindHttpException;
+import org.opensearch.http.CorsHandler;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.HttpTransportSettings;
+import org.opensearch.http.NullDispatcher;
+import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.RestChannel;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.telemetry.tracing.noop.NoopTracer;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.rest.FakeRestRequest;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.NettyAllocator;
+import org.opensearch.transport.reactor.SharedGroupFactory;
+import org.junit.After;
+import org.junit.Before;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.PoolArenaMetric;
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocatorMetric;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaderValues;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpUtil;
+import io.netty.handler.codec.http.HttpVersion;
+
+import static org.opensearch.core.rest.RestStatus.OK;
+import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
+import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Tests for the {@link ReactorNetty4HttpServerTransport} class.
+ */
+public class ReactorNetty4HttpServerTransportTests extends OpenSearchTestCase {
+
+    private NetworkService networkService;
+    private ThreadPool threadPool;
+    private MockBigArrays bigArrays;
+    private ClusterSettings clusterSettings;
+
+    @Before
+    public void setup() throws Exception {
+        networkService = new NetworkService(Collections.emptyList());
+        threadPool = new TestThreadPool("test");
+        bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
+        clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    }
+
+    @After
+    public void shutdown() throws Exception {
+        if (threadPool != null) {
+            threadPool.shutdownNow();
+        }
+        threadPool = null;
+        networkService = null;
+        bigArrays = null;
+        clusterSettings = null;
+    }
+
+    /**
+     * Test that {@link ReactorNetty4HttpServerTransport} supports the "Expect: 100-continue" HTTP header
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectContinueHeader() throws InterruptedException {
+        final Settings settings = createSettings();
+        final int contentLength = randomIntBetween(1, HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings).bytesAsInt());
+        runExpectHeaderTest(settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.CONTINUE);
+    }
+
+    /**
+     * Test that {@link ReactorNetty4HttpServerTransport} responds to a
+     * 100-continue expectation with too large a content-length
+     * with a 413 status.
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectContinueHeaderContentLengthTooLong() throws InterruptedException {
+        final String key = HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.getKey();
+        final int maxContentLength = randomIntBetween(1, 104857600);
+        final Settings settings = createBuilderWithPort().put(key, maxContentLength + "b").build();
+        final int contentLength = randomIntBetween(maxContentLength + 1, Integer.MAX_VALUE);
+        runExpectHeaderTest(settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE);
+    }
+
+    /**
+     * Test that {@link ReactorNetty4HttpServerTransport} responds to an unsupported expectation with a 417 status.
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectUnsupportedExpectation() throws InterruptedException {
+        Settings settings = createSettings();
+        runExpectHeaderTest(settings, "chocolate=yummy", 0, HttpResponseStatus.EXPECTATION_FAILED);
+    }
+
+    private void runExpectHeaderTest(
+        final Settings settings,
+        final String expectation,
+        final int contentLength,
+        final HttpResponseStatus expectedStatus
+    ) throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+                channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, new BytesArray("done")));
+            }
+
+            @Override
+            public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError();
+            }
+        };
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(settings),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+            try (ReactorHttpClient client = ReactorHttpClient.create()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
+                request.headers().set(HttpHeaderNames.EXPECT, expectation);
+                HttpUtil.setContentLength(request, contentLength);
+
+                // Reactor Netty 4 does not expose 100 CONTINUE response but instead just asks for content
+                final HttpContent continuationRequest = new DefaultHttpContent(Unpooled.EMPTY_BUFFER);
+                final FullHttpResponse continuationResponse = client.send(remoteAddress.address(), request, continuationRequest);
+                try {
+                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                } finally {
+                    continuationResponse.release();
+                }
+            }
+        }
+    }
+
+    public void testBindUnavailableAddress() {
+        Settings initialSettings = createSettings();
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                initialSettings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                new NullDispatcher(),
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+            Settings settings = Settings.builder()
+                .put("http.port", remoteAddress.getPort())
+                .put("network.host", remoteAddress.getAddress())
+                .build();
+            try (
+                ReactorNetty4HttpServerTransport otherTransport = new ReactorNetty4HttpServerTransport(
+                    settings,
+                    networkService,
+                    bigArrays,
+                    threadPool,
+                    xContentRegistry(),
+                    new NullDispatcher(),
+                    clusterSettings,
+                    new SharedGroupFactory(settings),
+                    NoopTracer.INSTANCE
+                )
+            ) {
+                BindHttpException bindHttpException = expectThrows(BindHttpException.class, otherTransport::start);
+                assertEquals("Failed to bind to " + NetworkAddress.format(remoteAddress.address()), bindHttpException.getMessage());
+            }
+        }
+    }
+
+    public void testBadRequest() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
+                throw new AssertionError();
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error("--> Unexpected bad request request");
+                throw new AssertionError(cause);
+            }
+        };
+
+        final Settings settings;
+        final int maxInitialLineLength;
+        final Setting httpMaxInitialLineLengthSetting = HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
+        if (randomBoolean()) {
+            maxInitialLineLength = httpMaxInitialLineLengthSetting.getDefault(Settings.EMPTY).bytesAsInt();
+            settings = createSettings();
+        } else {
+            maxInitialLineLength = randomIntBetween(1, 8192);
+            settings = createBuilderWithPort().put(httpMaxInitialLineLengthSetting.getKey(), maxInitialLineLength + "b").build();
+        }
+
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(settings),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            try (ReactorHttpClient client = ReactorHttpClient.create()) {
+                final String url = "/" + randomAlphaOfLength(maxInitialLineLength);
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.REQUEST_URI_TOO_LONG));
+                    assertThat(response.content().array().length, equalTo(0));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    public void testDispatchFailed() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                throw new RuntimeException("Bad things happen");
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error("--> Unexpected bad request request");
+                throw new AssertionError(cause);
+            }
+        };
+
+        final Settings settings = createSettings();
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(settings),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            try (ReactorHttpClient client = ReactorHttpClient.create()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+                    assertThat(response.content().array().length, equalTo(0));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    public void testLargeCompressedResponse() throws InterruptedException {
+        final String responseString = randomAlphaOfLength(4 * 1024 * 1024);
+        final String url = "/thing/";
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                if (url.equals(request.uri())) {
+                    channel.sendResponse(new BytesRestResponse(OK, responseString));
+                } else {
+                    logger.error("--> Unexpected successful uri [{}]", request.uri());
+                    throw new AssertionError();
+                }
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError();
+            }
+
+        };
+
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                Settings.EMPTY,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            try (ReactorHttpClient client = ReactorHttpClient.create()) {
+                DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
+                request.headers().add(HttpHeaderNames.ACCEPT_ENCODING, randomFrom("deflate", "gzip"));
+                long numOfHugeAllocations = getHugeAllocationCount();
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(getHugeAllocationCount(), equalTo(numOfHugeAllocations));
+                    assertThat(response.status(), equalTo(HttpResponseStatus.OK));
+                    byte[] bytes = new byte[response.content().readableBytes()];
+                    response.content().readBytes(bytes);
+                    assertThat(new String(bytes, StandardCharsets.UTF_8), equalTo(responseString));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    private long getHugeAllocationCount() {
+        long numOfHugAllocations = 0;
+        ByteBufAllocator allocator = NettyAllocator.getAllocator();
+        assert allocator instanceof NettyAllocator.NoDirectBuffers;
+        ByteBufAllocator delegate = ((NettyAllocator.NoDirectBuffers) allocator).getDelegate();
+        if (delegate instanceof PooledByteBufAllocator) {
+            PooledByteBufAllocatorMetric metric = ((PooledByteBufAllocator) delegate).metric();
+            numOfHugAllocations = metric.heapArenas().stream().mapToLong(PoolArenaMetric::numHugeAllocations).sum();
+        }
+        return numOfHugAllocations;
+    }
+
+    public void testCorsRequest() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
+                throw new AssertionError();
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError();
+            }
+
+        };
+
+        final Settings settings = createBuilderWithPort().put(SETTING_CORS_ENABLED.getKey(), true)
+            .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "test-cors.org")
+            .build();
+
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                new SharedGroupFactory(settings),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            // Test pre-flight request
+            try (ReactorHttpClient client = ReactorHttpClient.create()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/");
+                request.headers().add(CorsHandler.ORIGIN, "test-cors.org");
+                request.headers().add(CorsHandler.ACCESS_CONTROL_REQUEST_METHOD, "POST");
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.OK));
+                    assertThat(response.headers().get(CorsHandler.ACCESS_CONTROL_ALLOW_ORIGIN), equalTo("test-cors.org"));
+                    assertThat(response.headers().get(CorsHandler.VARY), equalTo(CorsHandler.ORIGIN));
+                    assertTrue(response.headers().contains(CorsHandler.DATE));
+                } finally {
+                    response.release();
+                }
+            }
+
+            // Test short-circuited request
+            try (ReactorHttpClient client = ReactorHttpClient.create()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+                request.headers().add(CorsHandler.ORIGIN, "google.com");
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.FORBIDDEN));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    public void testConnectTimeout() throws Exception {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
+                throw new AssertionError("Should not have received a dispatched request");
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError("Should not have received a dispatched request");
+            }
+
+        };
+
+        Settings settings = createBuilderWithPort().put(
+            HttpTransportSettings.SETTING_HTTP_CONNECT_TIMEOUT.getKey(),
+            new TimeValue(randomIntBetween(100, 300))
+        ).build();
+
+        NioEventLoopGroup group = new NioEventLoopGroup();
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                new SharedGroupFactory(settings),
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            final CountDownLatch channelClosedLatch = new CountDownLatch(1);
+
+            final Bootstrap clientBootstrap = new Bootstrap().option(ChannelOption.ALLOCATOR, NettyAllocator.getAllocator())
+                .channel(NioSocketChannel.class)
+                .handler(new ChannelInitializer() {
+
+                    @Override
+                    protected void initChannel(SocketChannel ch) {
+                        ch.pipeline().addLast(new ChannelHandlerAdapter() {
+                        });
+
+                    }
+                })
+                .group(group);
+            ChannelFuture connect = clientBootstrap.connect(remoteAddress.address());
+            connect.channel().closeFuture().addListener(future -> channelClosedLatch.countDown());
+
+            assertTrue("Channel should be closed due to read timeout", channelClosedLatch.await(1, TimeUnit.MINUTES));
+
+        } finally {
+            group.shutdownGracefully().await();
+        }
+    }
+
+    private Settings createSettings() {
+        return createBuilderWithPort().build();
+    }
+
+    private Settings.Builder createBuilderWithPort() {
+        return Settings.builder().put(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), getPortRange());
+    }
+}
diff --git a/server/build.gradle b/server/build.gradle
index c56f9d5aa288f..fa8a44ef6fc94 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -154,6 +154,10 @@ dependencies {
   // jcraft
   api "com.jcraft:jzlib:${versions.jzlib}"
 
+  // reactor
+  api "io.projectreactor:reactor-core:${versions.reactor}"
+  api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
+
   // protobuf
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
@@ -366,11 +370,13 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
             'com.google.protobuf.UnsafeUtil$MemoryAccessor',
             'org.apache.logging.log4j.core.util.internal.UnsafeUtil',
-            'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1'
+            'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1',
+            'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
     )
 }
 
 tasks.named("dependencyLicenses").configure {
+    mapping from: /reactor-.*/, to: 'reactor'
     mapping from: /lucene-.*/, to: 'lucene'
     dependencies = project.configurations.runtimeClasspath.fileCollection {
         it.group.startsWith('org.opensearch') == false ||
diff --git a/plugins/crypto-kms/licenses/reactive-streams-1.0.4.jar.sha1 b/server/licenses/reactive-streams-1.0.4.jar.sha1
similarity index 100%
rename from plugins/crypto-kms/licenses/reactive-streams-1.0.4.jar.sha1
rename to server/licenses/reactive-streams-1.0.4.jar.sha1
diff --git a/plugins/crypto-kms/licenses/reactive-streams-LICENSE.txt b/server/licenses/reactive-streams-LICENSE.txt
similarity index 100%
rename from plugins/crypto-kms/licenses/reactive-streams-LICENSE.txt
rename to server/licenses/reactive-streams-LICENSE.txt
diff --git a/plugins/discovery-ec2/licenses/reactive-streams-NOTICE.txt b/server/licenses/reactive-streams-NOTICE.txt
similarity index 100%
rename from plugins/discovery-ec2/licenses/reactive-streams-NOTICE.txt
rename to server/licenses/reactive-streams-NOTICE.txt
diff --git a/server/licenses/reactor-LICENSE.txt b/server/licenses/reactor-LICENSE.txt
new file mode 100644
index 0000000000000..e5583c184e67a
--- /dev/null
+++ b/server/licenses/reactor-LICENSE.txt
@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/reactive-streams-NOTICE.txt b/server/licenses/reactor-NOTICE.txt
similarity index 100%
rename from plugins/repository-azure/licenses/reactive-streams-NOTICE.txt
rename to server/licenses/reactor-NOTICE.txt
diff --git a/server/licenses/reactor-core-3.5.11.jar.sha1 b/server/licenses/reactor-core-3.5.11.jar.sha1
new file mode 100644
index 0000000000000..e5ffdbc8a7840
--- /dev/null
+++ b/server/licenses/reactor-core-3.5.11.jar.sha1
@@ -0,0 +1 @@
+db2299757f562261eb775d13658e86ff06f91e8a
\ No newline at end of file
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 5ab1f49949679..2bb81064c9c71 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -348,6 +348,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE,
                 HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH,
                 HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT,
+                HttpTransportSettings.SETTING_HTTP_CONNECT_TIMEOUT,
                 HttpTransportSettings.SETTING_HTTP_RESET_COOKIES,
                 HttpTransportSettings.OLD_SETTING_HTTP_TCP_NO_DELAY,
                 HttpTransportSettings.SETTING_HTTP_TCP_NO_DELAY,
diff --git a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java
index f16f06f414e28..621ef36692178 100644
--- a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java
+++ b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java
@@ -182,6 +182,14 @@ public final class HttpTransportSettings {
         Property.NodeScope
     );
 
+    // A default of 0 means that by default there is no connect timeout
+    public static final Setting SETTING_HTTP_CONNECT_TIMEOUT = Setting.timeSetting(
+        "http.connect_timeout",
+        new TimeValue(0),
+        new TimeValue(0),
+        Property.NodeScope
+    );
+
     // Tcp socket settings
 
     public static final Setting OLD_SETTING_HTTP_TCP_NO_DELAY = boolSetting(
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java
index 8490ee4fc39bc..b5ff30deecf5c 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java
@@ -175,6 +175,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import reactor.core.scheduler.Schedulers;
+
 import static java.util.Collections.emptyMap;
 import static org.opensearch.core.common.util.CollectionUtils.arrayAsArrayList;
 import static org.hamcrest.Matchers.empty;
@@ -225,6 +227,7 @@ public static void resetPortCounter() {
 
     @Override
     public void tearDown() throws Exception {
+        Schedulers.shutdownNow();
         FeatureFlagSetter.clear();
         super.tearDown();
     }

From c40014f8d37f27811362a3d6b3296cdc032b4399 Mon Sep 17 00:00:00 2001
From: Navneet Verma 
Date: Mon, 6 Nov 2023 13:55:27 -0800
Subject: [PATCH 273/388] Adding version condition while adding geoshape doc
 values to the index, to ensure backward compatibility. (#11095)

Signed-off-by: Navneet Verma 
---
 CHANGELOG.md                                  |  1 +
 ...AbstractGeoBucketAggregationIntegTest.java |  6 +-
 .../test/mixed_cluster/30_geoshape.yml        | 16 +++++
 .../test/old_cluster/30_geoshape.yml          | 28 +++++++++
 .../test/upgraded_cluster/30_geoshape.yml     | 61 +++++++++++++++++++
 .../index/mapper/GeoShapeFieldMapper.java     | 25 +++++++-
 6 files changed, 131 insertions(+), 6 deletions(-)
 create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_geoshape.yml
 create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_geoshape.yml
 create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_geoshape.yml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a71d2ff537834..435683442d4c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -139,6 +139,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
 - Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
+- Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095))
 
 ### Security
 
diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java
index 86d8ad2968e7f..7316847ac6046 100644
--- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java
+++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java
@@ -87,7 +87,7 @@ protected boolean forbidPrivateIndexSettings() {
      */
     protected void prepareGeoShapeIndexForAggregations(final Random random) throws Exception {
         expectedDocsCountForGeoShapes = new HashMap<>();
-        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
+        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
         final List geoshapes = new ArrayList<>();
         assertAcked(prepareCreate(GEO_SHAPE_INDEX_NAME).setSettings(settings).setMapping(GEO_SHAPE_FIELD_NAME, "type" + "=geo_shape"));
         boolean isShapeIntersectingBB = false;
@@ -136,7 +136,7 @@ protected void prepareSingleValueGeoPointIndex(final Random random) throws Excep
         expectedDocCountsForSingleGeoPoint = new HashMap<>();
         createIndex("idx_unmapped");
         final Settings settings = Settings.builder()
-            .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
             .put("index.number_of_shards", 4)
             .put("index.number_of_replicas", 0)
             .build();
@@ -160,7 +160,7 @@ protected void prepareSingleValueGeoPointIndex(final Random random) throws Excep
 
     protected void prepareMultiValuedGeoPointIndex(final Random random) throws Exception {
         multiValuedExpectedDocCountsGeoPoint = new HashMap<>();
-        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
+        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
         final List cities = new ArrayList<>();
         assertAcked(
             prepareCreate("multi_valued_idx").setSettings(settings)
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_geoshape.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_geoshape.yml
new file mode 100644
index 0000000000000..e669016cad98a
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_geoshape.yml
@@ -0,0 +1,16 @@
+---
+"Insert Document with geoshape field":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "geo_shape_index_old", "_id":191}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [101.0, 0.0] ]}}'
+          - '{"index": {"_index": "geo_shape_index_old", "_id":219}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [106.0, 0.0] ]}}'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: geo_shape_index_old
+  - match: { hits.total: 2 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_geoshape.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_geoshape.yml
new file mode 100644
index 0000000000000..30a39447905c0
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_geoshape.yml
@@ -0,0 +1,28 @@
+---
+"Create index with Geoshape field":
+  - do:
+      indices.create:
+        index: geo_shape_index_old
+        body:
+          settings:
+            index:
+              number_of_replicas: 2
+          mappings:
+            "properties":
+              "location":
+                "type": "geo_shape"
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "geo_shape_index_old", "_id":191}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [101.0, 0.0] ]}}'
+          - '{"index": {"_index": "geo_shape_index_old", "_id":219}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [106.0, 0.0] ]}}'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: geo_shape_index_old
+  - match: { hits.total: 2 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_geoshape.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_geoshape.yml
new file mode 100644
index 0000000000000..4c7b12a7f1909
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_geoshape.yml
@@ -0,0 +1,61 @@
+---
+"Validate we are able to index documents after upgrade":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "geo_shape_index_old", "_id":191}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [101.0, 0.0] ]}}'
+          - '{"index": {"_index": "geo_shape_index_old", "_id":219}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [106.0, 0.0] ]}}'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: geo_shape_index_old
+  - match: { hits.total: 2 }
+
+
+---
+"Create index with Geoshape field in new cluster":
+  - do:
+      indices.create:
+        index: geo_shape_index_new
+        body:
+          settings:
+            index:
+              number_of_replicas: 2
+          mappings:
+            "properties":
+              "location":
+                "type": "geo_shape"
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "geo_shape_index_new", "_id":191}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [101.0, 0.0] ]}}'
+          - '{"index": {"_index": "geo_shape_index_new", "_id":219}}'
+          - '{"name": "NEMO Science Museum","location": {"type": "envelope","coordinates": [ [100.0, 1.0], [106.0, 0.0] ]}}'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: geo_shape_index_new
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: geo_shape_index_new
+        body:
+          aggregations:
+            myaggregation:
+              geo_bounds:
+                field: "location"
+  - match: { hits.total: 2 }
+  - match: { aggregations.myaggregation.bounds.top_left.lat: 0.9999999823048711 }
+  - match: { aggregations.myaggregation.bounds.top_left.lon: 99.99999999068677 }
+  - match: { aggregations.myaggregation.bounds.bottom_right.lat: 0.0 }
+  - match: { aggregations.myaggregation.bounds.bottom_right.lon: 105.99999996833503 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java
index 4a4b2684b5f4c..b44b4b75549c3 100644
--- a/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/GeoShapeFieldMapper.java
@@ -31,12 +31,15 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.LatLonShape;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
+import org.opensearch.Version;
 import org.opensearch.common.Explicit;
 import org.opensearch.common.geo.GeometryParser;
 import org.opensearch.common.geo.ShapeRelation;
@@ -77,6 +80,7 @@
  * @opensearch.internal
  */
 public class GeoShapeFieldMapper extends AbstractShapeGeometryFieldMapper {
+    private static final Logger logger = LogManager.getLogger(GeoShapeFieldMapper.class);
     public static final String CONTENT_TYPE = "geo_shape";
     public static final FieldType FIELD_TYPE = new FieldType();
     static {
@@ -205,9 +209,24 @@ protected void addDocValuesFields(
         final List indexableFields,
         final ParseContext context
     ) {
-        Field[] fieldsArray = new Field[indexableFields.size()];
-        fieldsArray = indexableFields.toArray(fieldsArray);
-        context.doc().add(LatLonShape.createDocValueField(name, fieldsArray));
+        /*
+         * We are adding the doc values for GeoShape only if the index is created with 2.9 and above version of
+         * OpenSearch. If we don't do that after the upgrade of OpenSearch customers are not able to index documents
+         * with GeoShape fields. Github issue: https://github.com/opensearch-project/OpenSearch/issues/10958,
+         * https://github.com/opensearch-project/OpenSearch/issues/10795
+         */
+        if (context.indexSettings().getIndexVersionCreated().onOrAfter(Version.V_2_9_0)) {
+            Field[] fieldsArray = new Field[indexableFields.size()];
+            fieldsArray = indexableFields.toArray(fieldsArray);
+            context.doc().add(LatLonShape.createDocValueField(name, fieldsArray));
+        } else {
+            logger.warn(
+                "The index was created with Version : {}, for geoshape doc values to work index must be "
+                    + "created with OpenSearch Version : {} or above",
+                context.indexSettings().getIndexVersionCreated(),
+                Version.V_2_9_0
+            );
+        }
     }
 
     @Override

From 3bbc4952941c7a62f04b8aca3c973bf35677adb6 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Mon, 6 Nov 2023 14:52:50 -0800
Subject: [PATCH 274/388] Fix flaky test
 RemoteIndexShardtests.testRepicaCleansUpOldCommitsWhenReceivingNew (#11111)

Signed-off-by: Marc Handalian 
---
 .../index/shard/RemoteIndexShardTests.java    | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
index 20cec90d79e3e..dd92bfb47afdb 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java
@@ -32,6 +32,7 @@
 import org.opensearch.indices.replication.common.ReplicationFailedException;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.CorruptionUtils;
+import org.opensearch.test.junit.annotations.TestLogging;
 import org.hamcrest.MatcherAssert;
 import org.junit.Assert;
 
@@ -297,31 +298,39 @@ public void testPrimaryRestart_PrimaryHasExtraCommits() throws Exception {
         }
     }
 
+    @TestLogging(reason = "Getting trace logs from replication package", value = "org.opensearch.indices.replication:TRACE")
     public void testRepicaCleansUpOldCommitsWhenReceivingNew() throws Exception {
         final Path remotePath = createTempDir();
         try (ReplicationGroup shards = createGroup(1, getIndexSettings(), indexMapping, new NRTReplicationEngineFactory(), remotePath)) {
             shards.startAll();
             final IndexShard primary = shards.getPrimary();
             final IndexShard replica = shards.getReplicas().get(0);
+            final Store store = replica.store();
+            final SegmentInfos initialCommit = store.readLastCommittedSegmentsInfo();
             shards.indexDocs(1);
             flushShard(primary);
             replicateSegments(primary, shards.getReplicas());
+
             assertDocCount(primary, 1);
             assertDocCount(replica, 1);
-            assertEquals("segments_5", replica.store().readLastCommittedSegmentsInfo().getSegmentsFileName());
-            assertSingleSegmentFile(replica, "segments_5");
+            assertSingleSegmentFile(replica);
+            final SegmentInfos secondCommit = store.readLastCommittedSegmentsInfo();
+            assertTrue(secondCommit.getGeneration() > initialCommit.getGeneration());
 
             shards.indexDocs(1);
             primary.refresh("test");
             replicateSegments(primary, shards.getReplicas());
             assertDocCount(replica, 2);
-            assertSingleSegmentFile(replica, "segments_5");
+            assertSingleSegmentFile(replica);
+            assertEquals(store.readLastCommittedSegmentsInfo().getGeneration(), secondCommit.getGeneration());
 
             shards.indexDocs(1);
             flushShard(primary);
             replicateSegments(primary, shards.getReplicas());
             assertDocCount(replica, 3);
-            assertSingleSegmentFile(replica, "segments_6");
+            assertSingleSegmentFile(replica);
+            final SegmentInfos thirdCommit = store.readLastCommittedSegmentsInfo();
+            assertTrue(thirdCommit.getGeneration() > secondCommit.getGeneration());
 
             final Store.RecoveryDiff diff = Store.segmentReplicationDiff(primary.getSegmentMetadataMap(), replica.getSegmentMetadataMap());
             assertTrue(diff.missing.isEmpty());
@@ -571,11 +580,10 @@ protected void validateShardIdleWithNoReplicas(IndexShard primary) {
         assertFalse(primary.hasRefreshPending());
     }
 
-    private void assertSingleSegmentFile(IndexShard shard, String fileName) throws IOException {
+    private void assertSingleSegmentFile(IndexShard shard) throws IOException {
         final Set segmentsFileNames = Arrays.stream(shard.store().directory().listAll())
             .filter(file -> file.startsWith(IndexFileNames.SEGMENTS))
             .collect(Collectors.toSet());
         assertEquals("Expected a single segment file", 1, segmentsFileNames.size());
-        assertEquals(segmentsFileNames.stream().findFirst().get(), fileName);
     }
 }

From c206159b4e136ac9ddd96c0ec5eeb74465092088 Mon Sep 17 00:00:00 2001
From: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com>
Date: Mon, 6 Nov 2023 22:46:21 -0800
Subject: [PATCH 275/388] Fix flaky testMinScoreDisablesCountOptimization in
 query profile phase tests (#11104) (#11104)

Signed-off-by: Ticheng Lin 
---
 .../search/query/QueryProfilePhaseTests.java        | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java
index 28af8a63cfba8..62dcf54e25578 100644
--- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java
+++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java
@@ -336,12 +336,17 @@ public void testMinScoreDisablesCountOptimization() throws Exception {
         assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation);
         assertProfileData(context, "MatchAllDocsQuery", query -> {
             assertThat(query.getTimeBreakdown().keySet(), not(empty()));
-            assertThat(query.getTimeBreakdown().get("score"), greaterThanOrEqualTo(100L));
+            assertThat(query.getTimeBreakdown().get("score"), greaterThanOrEqualTo(1L));
             assertThat(query.getTimeBreakdown().get("score_count"), equalTo(1L));
             if (executor != null) {
-                assertThat(query.getTimeBreakdown().get("max_score"), greaterThanOrEqualTo(100L));
-                assertThat(query.getTimeBreakdown().get("min_score"), greaterThanOrEqualTo(100L));
-                assertThat(query.getTimeBreakdown().get("avg_score"), greaterThanOrEqualTo(100L));
+                long maxScore = query.getTimeBreakdown().get("max_score");
+                long minScore = query.getTimeBreakdown().get("min_score");
+                long avgScore = query.getTimeBreakdown().get("avg_score");
+                assertThat(maxScore, greaterThanOrEqualTo(1L));
+                assertThat(minScore, greaterThanOrEqualTo(1L));
+                assertThat(avgScore, greaterThanOrEqualTo(1L));
+                assertThat(maxScore, greaterThanOrEqualTo(avgScore));
+                assertThat(avgScore, greaterThanOrEqualTo(minScore));
                 assertThat(query.getTimeBreakdown().get("max_score_count"), equalTo(1L));
                 assertThat(query.getTimeBreakdown().get("min_score_count"), equalTo(1L));
                 assertThat(query.getTimeBreakdown().get("avg_score_count"), equalTo(1L));

From 145fd9ecd72ed4318a6e7ed3f49453cef6b9ac8b Mon Sep 17 00:00:00 2001
From: Jay Deng 
Date: Tue, 7 Nov 2023 04:40:20 -0800
Subject: [PATCH 276/388] Unmute FieldCapabilitiesIT.testWithIndexFilter for
 concurrent segment search (#11110)

Signed-off-by: Jay Deng 
---
 .../org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java  | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java
index 6b95405b3ebd4..f5d1b8234558e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fieldcaps/FieldCapabilitiesIT.java
@@ -244,10 +244,6 @@ public void testWithIndexAlias() {
     }
 
     public void testWithIndexFilter() throws InterruptedException {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10433",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         assertAcked(prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"));
         assertAcked(prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long"));
 

From 0ba5d5807860a799c76f0fb6e063e8c5bd458492 Mon Sep 17 00:00:00 2001
From: Peter Nied 
Date: Tue, 7 Nov 2023 11:27:16 -0600
Subject: [PATCH 277/388] Fix SuggestSearch.testSkipDuplicates by forceing
 refresh when indexing its test documents (#11068)

During the testSkipDuplicates its possible that not all documents were
fully indexed by the time the search with suggestions was created,
updating the indexing operations to refresh the index before
returning.

As its possible that did not fix the issue, I've added logging around
the test case to capture the state when the error occurred that can
assist in future troubleshooting.

Signed-off-by: Peter Nied 
---
 CHANGELOG.md                                               | 1 +
 .../search/suggest/CompletionSuggestSearchIT.java          | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 435683442d4c9..2011769858699 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -139,6 +139,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))
 - Fix some test methods in SimulatePipelineRequestParsingTests never run and fix test failure ([#10496](https://github.com/opensearch-project/OpenSearch/pull/10496))
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
+- Fix SuggestSearch.testSkipDuplicates by forceing refresh when indexing its test documents ([#11068](https://github.com/opensearch-project/OpenSearch/pull/11068))
 - Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095))
 
 ### Security
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java
index 30dba87f8ef5d..b342e6d35f0b4 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java
@@ -79,6 +79,7 @@
 import java.util.Set;
 
 import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.opensearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -1171,6 +1172,7 @@ public void testSkipDuplicates() throws Exception {
         createIndexAndMapping(mapping);
         int numDocs = randomIntBetween(10, 100);
         int numUnique = randomIntBetween(1, numDocs);
+        logger.info("Suggestion duplicate parameters: numDocs {} numUnique {}", numDocs, numUnique);
         List indexRequestBuilders = new ArrayList<>();
         int[] weights = new int[numUnique];
         Integer[] termIds = new Integer[numUnique];
@@ -1180,8 +1182,10 @@ public void testSkipDuplicates() throws Exception {
             int weight = randomIntBetween(0, 100);
             weights[id] = Math.max(weight, weights[id]);
             String suggestion = "suggestion-" + String.format(Locale.ENGLISH, "%03d", id);
+            logger.info("Creating {}, id {}, weight {}", suggestion, i, id, weight);
             indexRequestBuilders.add(
                 client().prepareIndex(INDEX)
+                    .setRefreshPolicy(WAIT_UNTIL)
                     .setSource(
                         jsonBuilder().startObject()
                             .startObject(FIELD)
@@ -1195,10 +1199,12 @@ public void testSkipDuplicates() throws Exception {
         indexRandom(true, indexRequestBuilders);
 
         Arrays.sort(termIds, Comparator.comparingInt(o -> weights[(int) o]).reversed().thenComparingInt(a -> (int) a));
+        logger.info("Expected terms id ordered {}", (Object[]) termIds);
         String[] expected = new String[numUnique];
         for (int i = 0; i < termIds.length; i++) {
             expected[i] = "suggestion-" + String.format(Locale.ENGLISH, "%03d", termIds[i]);
         }
+        logger.info("Expected suggestions field values {}", (Object[]) expected);
         CompletionSuggestionBuilder completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD)
             .prefix("sugg")
             .skipDuplicates(true)
@@ -1207,6 +1213,7 @@ public void testSkipDuplicates() throws Exception {
         SearchResponse searchResponse = client().prepareSearch(INDEX)
             .suggest(new SuggestBuilder().addSuggestion("suggestions", completionSuggestionBuilder))
             .get();
+        logger.info("Search Response with Suggestions {}", searchResponse);
         assertSuggestions(searchResponse, true, "suggestions", expected);
     }
 

From 4d8c228d34ccc3bc060112ce0c974981113f67ed Mon Sep 17 00:00:00 2001
From: Ankit Kala 
Date: Wed, 8 Nov 2023 14:36:54 +0530
Subject: [PATCH 278/388] Flaky RemoteClustersIT: Add assert busy to avoid race
 condition (#11057)

Signed-off-by: Ankit Kala 
---
 .../cluster/remote/test/RemoteClustersIT.java  | 18 +++++++++++-------
 .../transport/ProxyConnectionStrategy.java     |  2 +-
 .../transport/RemoteClusterService.java        |  2 +-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/RemoteClustersIT.java b/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/RemoteClustersIT.java
index dbea8db1a12fa..c38fcc468c673 100644
--- a/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/RemoteClustersIT.java
+++ b/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/RemoteClustersIT.java
@@ -42,11 +42,13 @@
 import org.opensearch.client.cluster.RemoteInfoRequest;
 import org.opensearch.client.indices.CreateIndexRequest;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 
 
 public class RemoteClustersIT extends AbstractMultiClusterRemoteTestCase {
@@ -112,7 +114,7 @@ public void testSniffModeConnectionFails() throws IOException {
         assertFalse(rci.isConnected());
     }
 
-    public void testHAProxyModeConnectionWorks() throws IOException {
+    public void testHAProxyModeConnectionWorks() throws Exception {
         String proxyAddress = "haproxy:9600";
         logger.info("Configuring remote cluster [{}]", proxyAddress);
         ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest().persistentSettings(Settings.builder()
@@ -121,12 +123,14 @@ public void testHAProxyModeConnectionWorks() throws IOException {
             .build());
         assertTrue(cluster1Client().cluster().putSettings(request, RequestOptions.DEFAULT).isAcknowledged());
 
-        RemoteConnectionInfo rci = cluster1Client().cluster().remoteInfo(new RemoteInfoRequest(), RequestOptions.DEFAULT).getInfos().get(0);
-        logger.info("Connection info: {}", rci);
-        if (!rci.isConnected()) {    
-            logger.info("Cluster health: {}", cluster1Client().cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT));
-        }
-        assertTrue(rci.isConnected());
+        assertBusy(() -> {
+            RemoteConnectionInfo rci = cluster1Client().cluster().remoteInfo(new RemoteInfoRequest(), RequestOptions.DEFAULT).getInfos().get(0);
+            logger.info("Connection info: {}", rci);
+            if (!rci.isConnected()) {
+                logger.info("Cluster health: {}", cluster1Client().cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT));
+            }
+            assertTrue(rci.isConnected());
+        }, 10, TimeUnit.SECONDS);
 
         assertEquals(2L, cluster1Client().search(
             new SearchRequest("haproxynosn:test2"), RequestOptions.DEFAULT).getHits().getTotalHits().value);
diff --git a/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java
index e914542023ad9..b4477edaba687 100644
--- a/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/opensearch/transport/ProxyConnectionStrategy.java
@@ -309,7 +309,7 @@ public void onResponse(Void v) {
 
                     @Override
                     public void onFailure(Exception e) {
-                        logger.debug(
+                        logger.error(
                             new ParameterizedMessage(
                                 "failed to open remote connection [remote cluster: {}, address: {}]",
                                 clusterAlias,
diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterService.java b/server/src/main/java/org/opensearch/transport/RemoteClusterService.java
index 35691cc5f8a1e..87786fb22f22e 100644
--- a/server/src/main/java/org/opensearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/opensearch/transport/RemoteClusterService.java
@@ -270,7 +270,7 @@ protected void updateRemoteCluster(String clusterAlias, Settings settings) {
             // are on the cluster state thread and our custom future implementation will throw an
             // assertion.
             if (latch.await(10, TimeUnit.SECONDS) == false) {
-                logger.warn("failed to connect to new remote cluster {} within {}", clusterAlias, TimeValue.timeValueSeconds(10));
+                logger.error("failed to connect to new remote cluster {} within {}", clusterAlias, TimeValue.timeValueSeconds(10));
             }
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();

From a782f4fb2b69f47cb180e29c3c9b4f2aea90415b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Vl=C4=8Dek?= 
Date: Wed, 8 Nov 2023 15:15:16 +0100
Subject: [PATCH 279/388] Deprecate CamelCase PathHierarchy tokenizer name
 (#10894)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Deprecate CamelCase PathHierarchy tokenizer name in favor to lowercase path_hierarchy.

Signed-off-by: Lukáš Vlček 
---
 CHANGELOG.md                                  |  1 +
 .../common/CommonAnalysisModulePlugin.java    | 25 +++++++++--
 .../PathHierarchyTokenizerFactoryTests.java   | 45 +++++++++++++++++++
 .../test/analysis-common/30_tokenizers.yml    |  9 +++-
 4 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2011769858699..772afcb0ea715 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -63,6 +63,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855))
 - Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
 - Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558))
+- Deprecate CamelCase `PathHierarchy` tokenizer name in favor to lowercase `path_hierarchy` ([#10894](https://github.com/opensearch-project/OpenSearch/pull/10894))
 
 
 ### Deprecated
diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java
index edb8c37c2dbdd..cf2736a8583d2 100644
--- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java
+++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java
@@ -394,7 +394,17 @@ public Map> getTokenizers() {
         // TODO deprecate and remove in API
         tokenizers.put("lowercase", XLowerCaseTokenizerFactory::new);
         tokenizers.put("path_hierarchy", PathHierarchyTokenizerFactory::new);
-        tokenizers.put("PathHierarchy", PathHierarchyTokenizerFactory::new);
+        tokenizers.put("PathHierarchy", (IndexSettings indexSettings, Environment environment, String name, Settings settings) -> {
+            // TODO Remove "PathHierarchy" tokenizer name in 4.0 and throw exception
+            if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_3_0_0)) {
+                deprecationLogger.deprecate(
+                    "PathHierarchy_tokenizer_deprecation",
+                    "The [PathHierarchy] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to [path_hierarchy] instead."
+                );
+            }
+            return new PathHierarchyTokenizerFactory(indexSettings, environment, name, settings);
+        });
         tokenizers.put("pattern", PatternTokenizerFactory::new);
         tokenizers.put("uax_url_email", UAX29URLEmailTokenizerFactory::new);
         tokenizers.put("whitespace", WhitespaceTokenizerFactory::new);
@@ -662,8 +672,17 @@ public List getPreConfiguredTokenizers() {
             }
             return new EdgeNGramTokenizer(NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE, NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
         }));
-        tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new));
-
+        tokenizers.add(PreConfiguredTokenizer.openSearchVersion("PathHierarchy", (version) -> {
+            // TODO Remove "PathHierarchy" tokenizer name in 4.0 and throw exception
+            if (version.onOrAfter(Version.V_3_0_0)) {
+                deprecationLogger.deprecate(
+                    "PathHierarchy_tokenizer_deprecation",
+                    "The [PathHierarchy] tokenizer name is deprecated and will be removed in a future version. "
+                        + "Please change the tokenizer name to [path_hierarchy] instead."
+                );
+            }
+            return new PathHierarchyTokenizer();
+        }));
         return tokenizers;
     }
 }
diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java
index 1fe7c582449ec..555d6c78b6ec5 100644
--- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/PathHierarchyTokenizerFactoryTests.java
@@ -35,16 +35,61 @@
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.Index;
+import org.opensearch.env.Environment;
+import org.opensearch.env.TestEnvironment;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.analysis.IndexAnalyzers;
+import org.opensearch.index.analysis.NamedAnalyzer;
+import org.opensearch.indices.analysis.AnalysisModule;
 import org.opensearch.test.IndexSettingsModule;
 import org.opensearch.test.OpenSearchTokenStreamTestCase;
+import org.opensearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.io.StringReader;
+import java.util.Collections;
 
 public class PathHierarchyTokenizerFactoryTests extends OpenSearchTokenStreamTestCase {
 
+    private IndexAnalyzers buildAnalyzers(Version version, String tokenizer) throws IOException {
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+            .put("index.analysis.analyzer.my_analyzer.tokenizer", tokenizer)
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+        return new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(new CommonAnalysisModulePlugin()))
+            .getAnalysisRegistry()
+            .build(idxSettings);
+    }
+
+    /**
+     * Test that deprecated "PathHierarchy" tokenizer name is still available via {@link CommonAnalysisModulePlugin} starting in 3.x.
+     */
+    public void testPreConfiguredTokenizer() throws IOException {
+
+        {
+            try (
+                IndexAnalyzers indexAnalyzers = buildAnalyzers(
+                    VersionUtils.randomVersionBetween(random(), Version.V_3_0_0, Version.CURRENT),
+                    "PathHierarchy"
+                )
+            ) {
+                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
+                assertNotNull(analyzer);
+                assertTokenStreamContents(analyzer.tokenStream("dummy", "/a/b/c"), new String[] { "/a", "/a/b", "/a/b/c" });
+                // Once LUCENE-12750 is fixed we can use the following testing method instead.
+                // Similar testing approach has been used for deprecation of (Edge)NGrams tokenizers as well.
+                // assertAnalyzesTo(analyzer, "/a/b/c", new String[] { "/a", "/a/b", "/a/b/c" });
+
+            }
+        }
+    }
+
     public void testDefaults() throws IOException {
         final Index index = new Index("test", "_na_");
         final Settings indexSettings = newAnalysisSettingsBuilder().build();
diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml
index 56ed2175df60a..179de835a4105 100644
--- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml
+++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml
@@ -298,6 +298,9 @@
 
 ---
 "path_hierarchy":
+    - skip:
+        features: "allowed_warnings"
+
     - do:
         indices.analyze:
           body:
@@ -312,6 +315,8 @@
     - match:  { detail.tokenizer.tokens.2.token: a/b/c }
 
     - do:
+        allowed_warnings:
+          - 'The [PathHierarchy] tokenizer name is deprecated and will be removed in a future version. Please change the tokenizer name to [path_hierarchy] instead.'
         indices.analyze:
           body:
             text: "a/b/c"
@@ -337,11 +342,13 @@
     - match:  { detail.tokenizer.tokens.2.token: a/b/c }
 
     - do:
+        allowed_warnings:
+          - 'The [PathHierarchy] tokenizer name is deprecated and will be removed in a future version. Please change the tokenizer name to [path_hierarchy] instead.'
         indices.analyze:
           body:
             text: "a/b/c"
             explain: true
-            tokenizer:  PathHierarchy
+            tokenizer: PathHierarchy
     - length: { detail.tokenizer.tokens: 3 }
     - match:  { detail.tokenizer.name: PathHierarchy }
     - match:  { detail.tokenizer.tokens.0.token: a }

From ff3a6a3af549ef8948c99b95500b4bf4f8be76f0 Mon Sep 17 00:00:00 2001
From: Ethan Emoto <70109490+e-emoto@users.noreply.github.com>
Date: Wed, 8 Nov 2023 07:53:41 -0800
Subject: [PATCH 280/388] Fixed test cases for concurrent search (#11119)

Fixed test cases
            Updated fix for TransportTwoNodesSearchIT test cases
            Updated test cases and muted flaky tests
            Move indexRandomForConcurrentSearch call into prepareData
            Remove changes from TransportTwoNodesSearchIT.java until future fix

Signed-off-by: Ethan Emoto 
---
 .../search/aggregations/pipeline/DerivativeIT.java |  1 +
 .../search/aggregations/pipeline/MovAvgIT.java     |  1 +
 .../search/basic/SearchRedStateIndexIT.java        |  1 +
 .../search/basic/SearchWhileCreatingIndexIT.java   |  1 +
 .../search/fetch/FetchSubPhasePluginIT.java        |  5 +++++
 .../functionscore/RandomScoreFunctionIT.java       |  3 +++
 .../org/opensearch/search/geo/GeoFilterIT.java     |  2 ++
 .../search/morelikethis/MoreLikeThisIT.java        | 14 +++++++++++++-
 8 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java
index 18484c8a60ed7..41bbffc13658b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java
@@ -693,6 +693,7 @@ public void testAvgMovavgDerivNPE() throws Exception {
         }
 
         refresh();
+        indexRandomForConcurrentSearch("movavg_npe");
 
         SearchResponse response = client().prepareSearch("movavg_npe")
             .addAggregation(
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java
index 2aad0d2d38901..8ad3107ac33ac 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java
@@ -1320,6 +1320,7 @@ public void testPredictWithNonEmptyBuckets() throws Exception {
                     .setSource(jsonBuilder().startObject().field(INTERVAL_FIELD, i).field(VALUE_FIELD2, 10).endObject())
             );
         }
+        indexRandomForConcurrentSearch("predict_non_empty");
 
         bulkBuilder.get();
         ensureSearchable();
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java
index 44c4981dfdb36..bd623ccdf2731 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java
@@ -153,6 +153,7 @@ private void buildRedIndex(int numShards) throws Exception {
             client().prepareIndex("test").setId("" + i).setSource("field1", "value1").get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         internalCluster().stopRandomDataNode();
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java
index 71af7215c4eb7..a5989b693d332 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java
@@ -106,6 +106,7 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas)
         }
         client().prepareIndex("test").setId(id).setSource("field", "test").get();
         RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").get();
+        indexRandomForConcurrentSearch("test");
         // at least one shard should be successful when refreshing
         assertThat(refreshResponse.getSuccessfulShards(), greaterThanOrEqualTo(1));
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java
index 86df25c4dad65..77ddc5bf636f0 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java
@@ -100,6 +100,10 @@ protected Collection> nodePlugins() {
 
     @SuppressWarnings("unchecked")
     public void testPlugin() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11112",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         client().admin()
             .indices()
             .prepareCreate("test")
@@ -119,6 +123,7 @@ public void testPlugin() throws Exception {
             .actionGet();
 
         client().admin().indices().prepareRefresh().get();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch()
             .setSource(new SearchSourceBuilder().ext(Collections.singletonList(new TermVectorsFetchBuilder("test"))))
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java
index 8f43cefd2d53b..54cfcb7e6a683 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java
@@ -207,6 +207,7 @@ public void testScoreAccessWithinScript() throws Exception {
                 .get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         Map params = new HashMap<>();
         params.put("factor", randomIntBetween(2, 4));
@@ -368,6 +369,7 @@ public void testScoreRange() throws Exception {
         }
         flush();
         refresh();
+        indexRandomForConcurrentSearch("test");
         int iters = scaledRandomIntBetween(10, 20);
         for (int i = 0; i < iters; ++i) {
             SearchResponse searchResponse = client().prepareSearch()
@@ -390,6 +392,7 @@ public void testSeeds() throws Exception {
             index("test", "type", "" + i, jsonBuilder().startObject().endObject());
         }
         flushAndRefresh();
+        indexRandomForConcurrentSearch("test");
 
         assertNoFailures(
             client().prepareSearch()
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java
index 00524c6e04707..ba519be04edff 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java
@@ -266,6 +266,7 @@ public void testShapeRelations() throws Exception {
 
         client().prepareIndex("shapes").setId("1").setSource(data, MediaTypeRegistry.JSON).get();
         client().admin().indices().prepareRefresh().get();
+        indexRandomForConcurrentSearch("shapes");
 
         // Point in polygon
         SearchResponse result = client().prepareSearch()
@@ -427,6 +428,7 @@ public void testBulk() throws Exception {
 
         client().admin().indices().prepareCreate("countries").setSettings(settings).setMapping(xContentBuilder).get();
         BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, xContentBuilder.contentType()).get();
+        indexRandomForConcurrentSearch("countries");
 
         for (BulkItemResponse item : bulk.getItems()) {
             assertFalse("unable to index data", item.isFailed());
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java b/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java
index 87435bb0bd09d..dc7c4e687c2fa 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java
@@ -125,6 +125,7 @@ public void testSimpleMoreLikeThis() throws Exception {
         client().index(indexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject()))
             .actionGet();
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("Running moreLikeThis");
         SearchResponse response = client().prepareSearch()
@@ -155,6 +156,7 @@ public void testSimpleMoreLikeThisWithTypes() throws Exception {
         client().index(indexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject()))
             .actionGet();
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("Running moreLikeThis");
         SearchResponse response = client().prepareSearch()
@@ -190,6 +192,7 @@ public void testMoreLikeThisForZeroTokensInOneOfTheAnalyzedFields() throws Excep
         ).actionGet();
 
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(
@@ -256,6 +259,7 @@ public void testMoreLikeThisWithAliases() throws Exception {
         client().index(indexRequest("test").id("4").source(jsonBuilder().startObject().field("text", "opensearch release").endObject()))
             .actionGet();
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("Running moreLikeThis on index");
         SearchResponse response = client().prepareSearch()
@@ -304,6 +308,7 @@ public void testMoreLikeThisWithAliasesInLikeDocuments() throws Exception {
         client().index(indexRequest(indexName).id("3").source(jsonBuilder().startObject().field("text", "opensearch index").endObject()))
             .actionGet();
         refresh(indexName);
+        indexRandomForConcurrentSearch(indexName);
 
         SearchResponse response = client().prepareSearch()
             .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item(aliasName, "1") }).minTermFreq(1).minDocFreq(1))
@@ -321,6 +326,7 @@ public void testMoreLikeThisIssue2197() throws Exception {
             .get();
         client().admin().indices().prepareRefresh("foo").get();
         assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN));
+        indexRandomForConcurrentSearch("foo");
 
         SearchResponse response = client().prepareSearch()
             .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1") }))
@@ -344,6 +350,7 @@ public void testMoreLikeWithCustomRouting() throws Exception {
             .setRouting("2")
             .get();
         client().admin().indices().prepareRefresh("foo").get();
+        indexRandomForConcurrentSearch("foo");
 
         SearchResponse response = client().prepareSearch()
             .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("2") }))
@@ -368,6 +375,7 @@ public void testMoreLikeThisIssueRoutingNotSerialized() throws Exception {
             .setRouting("4000")
             .get();
         client().admin().indices().prepareRefresh("foo").get();
+        indexRandomForConcurrentSearch("foo");
         SearchResponse response = client().prepareSearch()
             .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("4000") }))
             .get();
@@ -401,6 +409,7 @@ public void testNumericField() throws Exception {
             .get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // Implicit list of fields -> ignore numeric fields
         SearchResponse searchResponse = client().prepareSearch()
@@ -508,6 +517,7 @@ public void testMoreLikeThisWithFieldAlias() throws Exception {
         index("test", "_doc", "1", "text", "lucene");
         index("test", "_doc", "2", "text", "lucene release");
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         Item item = new Item("test", "1");
         QueryBuilder query = QueryBuilders.moreLikeThisQuery(new String[] { "alias" }, null, new Item[] { item })
@@ -548,6 +558,7 @@ public void testSimpleMoreLikeInclude() throws Exception {
                 .source(jsonBuilder().startObject().field("text", "Lucene has been ported to other programming languages").endObject())
         ).actionGet();
         client().admin().indices().refresh(refreshRequest()).actionGet();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("Running More Like This with include true");
         SearchResponse response = client().prepareSearch()
@@ -832,11 +843,12 @@ public void testSelectFields() throws IOException, ExecutionException, Interrupt
         assertHitCount(response, 1);
     }
 
-    public void testWithRouting() throws IOException {
+    public void testWithRouting() throws IOException, InterruptedException {
         client().prepareIndex("index").setId("1").setRouting("3").setSource("text", "this is a document").get();
         client().prepareIndex("index").setId("2").setRouting("1").setSource("text", "this is another document").get();
         client().prepareIndex("index").setId("3").setRouting("4").setSource("text", "this is yet another document").get();
         refresh("index");
+        indexRandomForConcurrentSearch("index");
 
         Item item = new Item("index", "2").routing("1");
         MoreLikeThisQueryBuilder moreLikeThisQueryBuilder = new MoreLikeThisQueryBuilder(

From 0a9dfec4670ca8cab2bd62d79d474acbfc957b6a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Nov 2023 14:52:29 -0500
Subject: [PATCH 281/388] Bump aws-actions/configure-aws-credentials from 2 to
 4 (#10504)

* Bump aws-actions/configure-aws-credentials from 2 to 4

Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 2 to 4.
- [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases)
- [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md)
- [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/v2...v4)

---
updated-dependencies:
- dependency-name: aws-actions/configure-aws-credentials
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 .github/workflows/lucene-snapshots.yml        | 2 +-
 .github/workflows/publish-maven-snapshots.yml | 2 +-
 CHANGELOG.md                                  | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/lucene-snapshots.yml b/.github/workflows/lucene-snapshots.yml
index 76981276fe085..ec5893ea546a9 100644
--- a/.github/workflows/lucene-snapshots.yml
+++ b/.github/workflows/lucene-snapshots.yml
@@ -50,7 +50,7 @@ jobs:
         run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ steps.version.outputs.REVISION }}
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
           aws-region: us-west-2
diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml
index 8c08df269a999..93bbfb8bbeab8 100644
--- a/.github/workflows/publish-maven-snapshots.yml
+++ b/.github/workflows/publish-maven-snapshots.yml
@@ -26,7 +26,7 @@ jobs:
           java-version: 17
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
           aws-region: us-east-1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 772afcb0ea715..8ff0d4e3fd285 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -115,6 +115,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
 - Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.1 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858), [#11000](https://github.com/opensearch-project/OpenSearch/pull/11000))
+- Bump `aws-actions/configure-aws-credentials` from 2 to 4 ([#10504](https://github.com/opensearch-project/OpenSearch/pull/10504))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -146,4 +147,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file

From 61a598bad7ce095219790e78196b1ae5b7428316 Mon Sep 17 00:00:00 2001
From: Craig Perkins 
Date: Thu, 9 Nov 2023 12:59:13 -0500
Subject: [PATCH 282/388] [Refactor] Use iterative approach to evaluate
 Regex.simpleMatch (#11060)

* Remove adjacent duplicates to optimize regex before processing

Signed-off-by: Craig Perkins 

* Add tests

Signed-off-by: Craig Perkins 

* Rename to removeAdjacentDuplicates

Signed-off-by: Craig Perkins 

* Add additional test case

Signed-off-by: Craig Perkins 

* Add CHANGELOG entry

Signed-off-by: Craig Perkins 

* Throw IllegalArgumentException if input is invalid

Signed-off-by: Craig Perkins 

* Add a space

Signed-off-by: Craig Perkins 

* Change error message

Signed-off-by: Craig Perkins 

* Use iterative solution with 2 pointers

Signed-off-by: Craig Perkins 

* Remove unused method

Signed-off-by: Craig Perkins 

* add tests and changelog

Signed-off-by: Stephen Crawford 

* remove bad push

Signed-off-by: Stephen Crawford 

* Update CHANGELOG.md

Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>

* Update CHANGELOG.md

Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>

* spotless

Signed-off-by: Stephen Crawford 

* Update comment

Signed-off-by: Craig Perkins 

* re-run CI

Signed-off-by: Craig Perkins 

---------

Signed-off-by: Craig Perkins 
Signed-off-by: Craig Perkins 
Signed-off-by: Stephen Crawford 
Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>
Co-authored-by: Stephen Crawford 
Co-authored-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com>
---
 CHANGELOG.md                                  |  3 +-
 .../org/opensearch/common/regex/Regex.java    | 54 +++++++++----------
 .../opensearch/common/regex/RegexTests.java   | 15 ++++++
 3 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8ff0d4e3fd285..e8f27b9323306 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -129,6 +129,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
 - Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
+- Use iterative approach to evaluate Regex.simpleMatch ([#11060](https://github.com/opensearch-project/OpenSearch/pull/11060))
 
 ### Deprecated
 
@@ -147,4 +148,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
\ No newline at end of file
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
diff --git a/server/src/main/java/org/opensearch/common/regex/Regex.java b/server/src/main/java/org/opensearch/common/regex/Regex.java
index 396af77c8a751..323b460af62df 100644
--- a/server/src/main/java/org/opensearch/common/regex/Regex.java
+++ b/server/src/main/java/org/opensearch/common/regex/Regex.java
@@ -129,35 +129,35 @@ public static boolean simpleMatch(String pattern, String str, boolean caseInsens
     }
 
     private static boolean simpleMatchWithNormalizedStrings(String pattern, String str) {
-        final int firstIndex = pattern.indexOf('*');
-        if (firstIndex == -1) {
-            return pattern.equals(str);
-        }
-        if (firstIndex == 0) {
-            if (pattern.length() == 1) {
-                return true;
-            }
-            final int nextIndex = pattern.indexOf('*', firstIndex + 1);
-            if (nextIndex == -1) {
-                // str.endsWith(pattern.substring(1)), but avoiding the construction of pattern.substring(1):
-                return str.regionMatches(str.length() - pattern.length() + 1, pattern, 1, pattern.length() - 1);
-            } else if (nextIndex == 1) {
-                // Double wildcard "**" - skipping the first "*"
-                return simpleMatchWithNormalizedStrings(pattern.substring(1), str);
-            }
-            final String part = pattern.substring(1, nextIndex);
-            int partIndex = str.indexOf(part);
-            while (partIndex != -1) {
-                if (simpleMatchWithNormalizedStrings(pattern.substring(nextIndex), str.substring(partIndex + part.length()))) {
-                    return true;
-                }
-                partIndex = str.indexOf(part, partIndex + 1);
+        int sIdx = 0, pIdx = 0, match = 0, wildcardIdx = -1;
+        while (sIdx < str.length()) {
+            // both chars matching, incrementing both pointers
+            if (pIdx < pattern.length() && str.charAt(sIdx) == pattern.charAt(pIdx)) {
+                sIdx++;
+                pIdx++;
+            } else if (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
+                // wildcard found, only incrementing pattern pointer
+                wildcardIdx = pIdx;
+                match = sIdx;
+                pIdx++;
+            } else if (wildcardIdx != -1) {
+                // last pattern pointer was a wildcard, incrementing string pointer
+                pIdx = wildcardIdx + 1;
+                match++;
+                sIdx = match;
+            } else {
+                // current pattern pointer is not a wildcard, last pattern pointer was also not a wildcard
+                // characters do not match
+                return false;
             }
-            return false;
         }
-        return str.regionMatches(0, pattern, 0, firstIndex)
-            && (firstIndex == pattern.length() - 1 // only wildcard in pattern is at the end, so no need to look at the rest of the string
-                || simpleMatchWithNormalizedStrings(pattern.substring(firstIndex), str.substring(firstIndex)));
+
+        // check for remaining characters in pattern
+        while (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
+            pIdx++;
+        }
+
+        return pIdx == pattern.length();
     }
 
     /**
diff --git a/server/src/test/java/org/opensearch/common/regex/RegexTests.java b/server/src/test/java/org/opensearch/common/regex/RegexTests.java
index b92fcdad56d74..21d3cb2df8f61 100644
--- a/server/src/test/java/org/opensearch/common/regex/RegexTests.java
+++ b/server/src/test/java/org/opensearch/common/regex/RegexTests.java
@@ -96,7 +96,22 @@ public void testDoubleWildcardMatch() {
         assertFalse(Regex.simpleMatch("fff**ddd", "fffabcdd"));
         assertTrue(Regex.simpleMatch("fff*******ddd", "fffabcddd"));
         assertTrue(Regex.simpleMatch("fff*******ddd", "FffAbcdDd", true));
+        assertFalse(Regex.simpleMatch("fff*******ddd", "FffAbcdDd", false));
         assertFalse(Regex.simpleMatch("fff******ddd", "fffabcdd"));
+        assertTrue(Regex.simpleMatch("abCDefGH******ddd", "abCDefGHddd", false));
+        assertTrue(Regex.simpleMatch("******", "a"));
+        assertTrue(Regex.simpleMatch("***WILDcard***", "aaaaaaaaWILDcardZZZZZZ", false));
+        assertFalse(Regex.simpleMatch("***xxxxx123456789xxxxxx***", "xxxxxabcdxxxxx", false));
+        assertFalse(Regex.simpleMatch("***xxxxxabcdxxxxx***", "xxxxxABCDxxxxx", false));
+        assertTrue(Regex.simpleMatch("***xxxxxabcdxxxxx***", "xxxxxABCDxxxxx", true));
+        assertTrue(Regex.simpleMatch("**stephenIsSuperCool**", "ItIsTrueThatStephenIsSuperCoolSoYouShouldLetThisIn", true));
+        assertTrue(
+            Regex.simpleMatch(
+                "**w**X**y**Z**",
+                "abcdeFGHIJKLMNOPqrstuvwabcdeFGHIJKLMNOPqrstuvwXabcdeFGHIJKLMNOPqrstuvwXyabcdeFGHIJKLMNOPqrstuvwXyZ",
+                false
+            )
+        );
     }
 
     public void testSimpleMatch() {

From 3eda422ae1979d3e1e7b158abab7d179f919f868 Mon Sep 17 00:00:00 2001
From: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com>
Date: Thu, 9 Nov 2023 13:37:32 -0800
Subject: [PATCH 283/388] Fix slice collectors to leaves association with post
 filter (#11134)

* Fix slice collectors to leaves association with profile enabled (#11134)

Signed-off-by: Ticheng Lin 

* Update server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java

Co-authored-by: Andriy Redko 
Signed-off-by: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com>
Signed-off-by: Ticheng Lin 

---------

Signed-off-by: Ticheng Lin 
Signed-off-by: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com>
Co-authored-by: Andriy Redko 
---
 .../opensearch/common/lucene/search/FilteredCollector.java   | 4 ++++
 .../profile/query/ConcurrentQueryProfileBreakdown.java       | 5 ++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/common/lucene/search/FilteredCollector.java b/server/src/main/java/org/opensearch/common/lucene/search/FilteredCollector.java
index 0d7a8866f7788..b5c0e84a10308 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/FilteredCollector.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/FilteredCollector.java
@@ -40,6 +40,7 @@
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.opensearch.common.lucene.Lucene;
+import org.opensearch.search.profile.query.ProfileWeight;
 
 import java.io.IOException;
 
@@ -64,6 +65,9 @@ public Collector getCollector() {
 
     @Override
     public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+        if (filter instanceof ProfileWeight) {
+            ((ProfileWeight) filter).associateCollectorToLeaves(context, collector);
+        }
         final ScorerSupplier filterScorerSupplier = filter.scorerSupplier(context);
         final LeafCollector in = collector.getLeafCollector(context);
         final Bits bits = Lucene.asSequentialAccessBits(context.reader().maxDoc(), filterScorerSupplier);
diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
index 59ef01f9f947a..e1d41227a22f7 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
@@ -83,7 +83,10 @@ public Map toBreakdownMap() {
             // creates a new weight and breakdown map for each rewritten query. This new breakdown map captures the timing information for
             // the new rewritten query. The sliceCollectorsToLeaves is empty because this breakdown for rewritten query gets created later
             // in search leaf path which doesn't have collector. Also, this is not needed since this breakdown is per leaf and there is no
-            // concurrency involved. An empty sliceCollectorsToLeaves could also happen in the case of early termination.
+            // concurrency involved.
+            assert contexts.size() == 1 : "Unexpected size: "
+                + contexts.size()
+                + " of leaves breakdown in ConcurrentQueryProfileBreakdown of rewritten query for a leaf.";
             AbstractProfileBreakdown breakdown = contexts.values().iterator().next();
             queryNodeTime = breakdown.toNodeTime() + createWeightTime;
             maxSliceNodeTime = 0L;

From 0d54c1600b16081c15902a095111a9a530767099 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Thu, 9 Nov 2023 18:41:19 -0500
Subject: [PATCH 284/388] [BWC and API enforcement] Decorate the existing APIs
 with proper annotations (part 2) (#10950)

Signed-off-by: Andriy Redko 
---
 .../opensearch/common/CheckedBiConsumer.java  |  5 ++++-
 .../opensearch/common/CheckedFunction.java    |  3 +++
 .../org/opensearch/common/TriFunction.java    |  5 ++++-
 .../org/opensearch/common/collect/Tuple.java  |  5 ++++-
 .../common/crypto/MasterKeyProvider.java      |  5 +++++
 .../common/io/InputStreamContainer.java       |  5 ++++-
 .../opensearch/OpenSearchParseException.java  |  4 +++-
 .../core/action/NotifyOnceListener.java       |  5 ++++-
 .../org/opensearch/core/common/text/Text.java |  4 +++-
 .../common/transport/TransportAddress.java    |  4 +++-
 .../tasks/resourcetracker/ResourceStats.java  |  5 ++++-
 .../resourcetracker/ResourceStatsType.java    |  5 ++++-
 .../resourcetracker/ResourceUsageInfo.java    |  4 +++-
 .../resourcetracker/ResourceUsageMetric.java  |  5 ++++-
 .../resourcetracker/TaskResourceStats.java    |  4 +++-
 .../resourcetracker/TaskResourceUsage.java    |  4 +++-
 .../resourcetracker/TaskThreadUsage.java      |  4 +++-
 .../resourcetracker/ThreadResourceInfo.java   |  5 ++++-
 .../core/xcontent/AbstractObjectParser.java   |  4 +++-
 .../core/xcontent/ContextParser.java          |  5 +++++
 .../core/xcontent/DeprecationHandler.java     |  5 +++++
 .../opensearch/core/xcontent/ToXContent.java  |  7 ++++--
 .../opensearch/core/xcontent/XContent.java    |  5 +++++
 .../core/xcontent/XContentParser.java         |  4 +++-
 .../common/xcontent/XContentType.java         |  3 +++
 .../OpenSearchGenerationException.java        |  4 +++-
 .../ActionRequestValidationException.java     |  2 ++
 .../opensearch/action/DocWriteRequest.java    |  7 +++++-
 .../opensearch/action/DocWriteResponse.java   |  7 +++++-
 .../org/opensearch/action/IndicesRequest.java |  4 +++-
 .../ClusterAllocationExplainRequest.java      |  4 +++-
 ...lusterAllocationExplainRequestBuilder.java |  4 +++-
 .../ClusterAllocationExplainResponse.java     |  4 +++-
 .../admin/cluster/crypto/CryptoSettings.java  |  4 +++-
 .../DeleteDecommissionStateRequest.java       |  4 +++-
 ...DeleteDecommissionStateRequestBuilder.java |  4 +++-
 .../DeleteDecommissionStateResponse.java      |  4 +++-
 .../get/GetDecommissionStateRequest.java      |  4 +++-
 .../GetDecommissionStateRequestBuilder.java   |  4 +++-
 .../get/GetDecommissionStateResponse.java     |  4 +++-
 .../awareness/put/DecommissionRequest.java    |  4 +++-
 .../put/DecommissionRequestBuilder.java       |  4 +++-
 .../awareness/put/DecommissionResponse.java   |  4 +++-
 .../cluster/health/ClusterHealthRequest.java  |  7 ++++--
 .../health/ClusterHealthRequestBuilder.java   |  4 +++-
 .../cluster/health/ClusterHealthResponse.java |  4 +++-
 .../hotthreads/NodesHotThreadsRequest.java    |  4 +++-
 .../NodesHotThreadsRequestBuilder.java        |  4 +++-
 .../hotthreads/NodesHotThreadsResponse.java   |  4 +++-
 .../cluster/node/info/NodesInfoRequest.java   |  4 +++-
 .../node/info/NodesInfoRequestBuilder.java    |  4 +++-
 .../cluster/node/info/NodesInfoResponse.java  |  4 +++-
 ...desReloadSecureSettingsRequestBuilder.java |  4 +++-
 .../cluster/node/stats/NodesStatsRequest.java |  4 +++-
 .../node/stats/NodesStatsRequestBuilder.java  |  4 +++-
 .../node/stats/NodesStatsResponse.java        |  4 +++-
 .../node/tasks/cancel/CancelTasksRequest.java |  4 +++-
 .../cancel/CancelTasksRequestBuilder.java     |  4 +++-
 .../tasks/cancel/CancelTasksResponse.java     |  4 +++-
 .../node/tasks/get/GetTaskRequest.java        |  4 +++-
 .../node/tasks/get/GetTaskRequestBuilder.java |  4 +++-
 .../node/tasks/get/GetTaskResponse.java       |  4 +++-
 .../node/tasks/list/ListTasksRequest.java     |  4 +++-
 .../tasks/list/ListTasksRequestBuilder.java   |  4 +++-
 .../node/tasks/list/ListTasksResponse.java    |  4 +++-
 .../cluster/node/tasks/list/TaskGroup.java    |  7 ++++--
 .../cluster/node/usage/NodesUsageRequest.java |  4 +++-
 .../node/usage/NodesUsageRequestBuilder.java  |  4 +++-
 .../node/usage/NodesUsageResponse.java        |  4 +++-
 .../restore/RestoreRemoteStoreRequest.java    |  4 +++-
 .../restore/RestoreRemoteStoreResponse.java   |  4 +++-
 .../remotestore/stats/RemoteStoreStats.java   |  4 +++-
 .../stats/RemoteStoreStatsRequest.java        |  4 +++-
 .../stats/RemoteStoreStatsRequestBuilder.java |  4 +++-
 .../stats/RemoteStoreStatsResponse.java       |  4 +++-
 .../cleanup/CleanupRepositoryRequest.java     |  4 +++-
 .../CleanupRepositoryRequestBuilder.java      |  4 +++-
 .../cleanup/CleanupRepositoryResponse.java    |  4 +++-
 .../delete/DeleteRepositoryRequest.java       |  4 +++-
 .../DeleteRepositoryRequestBuilder.java       |  4 +++-
 .../get/GetRepositoriesRequest.java           |  4 +++-
 .../get/GetRepositoriesRequestBuilder.java    |  4 +++-
 .../get/GetRepositoriesResponse.java          |  4 +++-
 .../put/PutRepositoryRequest.java             |  4 +++-
 .../put/PutRepositoryRequestBuilder.java      |  4 +++-
 .../verify/VerifyRepositoryRequest.java       |  4 +++-
 .../VerifyRepositoryRequestBuilder.java       |  4 +++-
 .../verify/VerifyRepositoryResponse.java      |  7 ++++--
 .../reroute/ClusterRerouteRequest.java        |  4 +++-
 .../reroute/ClusterRerouteRequestBuilder.java |  4 +++-
 .../reroute/ClusterRerouteResponse.java       |  4 +++-
 .../ClusterUpdateSettingsRequest.java         |  4 +++-
 .../ClusterUpdateSettingsRequestBuilder.java  |  4 +++-
 .../ClusterUpdateSettingsResponse.java        |  4 +++-
 .../shards/ClusterSearchShardsGroup.java      |  4 +++-
 .../shards/ClusterSearchShardsRequest.java    |  4 +++-
 .../ClusterSearchShardsRequestBuilder.java    |  4 +++-
 .../shards/ClusterSearchShardsResponse.java   |  4 +++-
 .../ClusterDeleteWeightedRoutingRequest.java  |  4 +++-
 ...erDeleteWeightedRoutingRequestBuilder.java |  4 +++-
 .../ClusterDeleteWeightedRoutingResponse.java |  4 +++-
 .../get/ClusterGetWeightedRoutingRequest.java |  4 +++-
 ...usterGetWeightedRoutingRequestBuilder.java |  4 +++-
 .../ClusterGetWeightedRoutingResponse.java    |  4 +++-
 .../put/ClusterPutWeightedRoutingRequest.java |  4 +++-
 ...usterPutWeightedRoutingRequestBuilder.java |  4 +++-
 .../ClusterPutWeightedRoutingResponse.java    |  4 +++-
 .../snapshots/clone/CloneSnapshotRequest.java |  4 +++-
 .../clone/CloneSnapshotRequestBuilder.java    |  4 +++-
 .../create/CreateSnapshotRequest.java         |  4 +++-
 .../create/CreateSnapshotRequestBuilder.java  |  4 +++-
 .../create/CreateSnapshotResponse.java        |  4 +++-
 .../delete/DeleteSnapshotRequest.java         |  4 +++-
 .../delete/DeleteSnapshotRequestBuilder.java  |  4 +++-
 .../snapshots/get/GetSnapshotsRequest.java    |  4 +++-
 .../get/GetSnapshotsRequestBuilder.java       |  4 +++-
 .../snapshots/get/GetSnapshotsResponse.java   |  4 +++-
 .../restore/RestoreSnapshotRequest.java       |  7 +++++-
 .../RestoreSnapshotRequestBuilder.java        |  4 +++-
 .../restore/RestoreSnapshotResponse.java      |  4 +++-
 .../status/SnapshotIndexShardStage.java       |  5 ++++-
 .../status/SnapshotIndexShardStatus.java      |  4 +++-
 .../snapshots/status/SnapshotIndexStatus.java |  4 +++-
 .../snapshots/status/SnapshotShardsStats.java |  4 +++-
 .../snapshots/status/SnapshotStats.java       |  4 +++-
 .../status/SnapshotsStatusRequest.java        |  4 +++-
 .../status/SnapshotsStatusRequestBuilder.java |  4 +++-
 .../status/SnapshotsStatusResponse.java       |  4 +++-
 .../cluster/state/ClusterStateRequest.java    |  4 +++-
 .../state/ClusterStateRequestBuilder.java     |  4 +++-
 .../cluster/state/ClusterStateResponse.java   |  4 +++-
 .../admin/cluster/stats/AnalysisStats.java    |  4 +++-
 .../cluster/stats/ClusterStatsIndices.java    |  7 ++++--
 .../cluster/stats/ClusterStatsNodes.java      | 19 +++++++++++-----
 .../cluster/stats/ClusterStatsRequest.java    |  4 +++-
 .../stats/ClusterStatsRequestBuilder.java     |  4 +++-
 .../cluster/stats/ClusterStatsResponse.java   |  4 +++-
 .../cluster/stats/IndexFeatureStats.java      |  4 +++-
 .../admin/cluster/stats/MappingStats.java     |  4 +++-
 .../DeleteStoredScriptRequest.java            |  4 +++-
 .../DeleteStoredScriptRequestBuilder.java     |  4 +++-
 .../storedscripts/GetStoredScriptRequest.java |  4 +++-
 .../GetStoredScriptRequestBuilder.java        |  4 +++-
 .../GetStoredScriptResponse.java              |  4 +++-
 .../storedscripts/PutStoredScriptRequest.java |  4 +++-
 .../PutStoredScriptRequestBuilder.java        |  4 +++-
 .../tasks/PendingClusterTasksRequest.java     |  4 +++-
 .../PendingClusterTasksRequestBuilder.java    |  4 +++-
 .../tasks/PendingClusterTasksResponse.java    |  4 +++-
 .../action/admin/indices/alias/Alias.java     |  4 +++-
 .../indices/alias/IndicesAliasesRequest.java  | 10 ++++++---
 .../alias/IndicesAliasesRequestBuilder.java   |  4 +++-
 .../indices/alias/get/GetAliasesRequest.java  |  4 +++-
 .../alias/get/GetAliasesRequestBuilder.java   |  4 +++-
 .../indices/alias/get/GetAliasesResponse.java |  4 +++-
 .../admin/indices/analyze/AnalyzeAction.java  | 22 +++++++++++++------
 .../analyze/AnalyzeRequestBuilder.java        |  4 +++-
 .../cache/clear/ClearIndicesCacheRequest.java |  4 +++-
 .../ClearIndicesCacheRequestBuilder.java      |  4 +++-
 .../clear/ClearIndicesCacheResponse.java      |  4 +++-
 .../indices/close/CloseIndexRequest.java      |  4 +++-
 .../close/CloseIndexRequestBuilder.java       |  4 +++-
 .../indices/close/CloseIndexResponse.java     | 13 +++++++----
 .../indices/create/CreateIndexRequest.java    |  4 +++-
 .../create/CreateIndexRequestBuilder.java     |  4 +++-
 .../indices/create/CreateIndexResponse.java   |  4 +++-
 .../delete/DeleteDanglingIndexRequest.java    |  4 +++-
 .../ImportDanglingIndexRequest.java           |  4 +++-
 .../list/ListDanglingIndicesRequest.java      |  4 +++-
 .../list/ListDanglingIndicesResponse.java     |  4 +++-
 .../datastream/CreateDataStreamAction.java    |  7 ++++--
 .../datastream/DeleteDataStreamAction.java    |  7 ++++--
 .../datastream/GetDataStreamAction.java       | 13 +++++++----
 .../indices/delete/DeleteIndexRequest.java    |  4 +++-
 .../delete/DeleteIndexRequestBuilder.java     |  4 +++-
 .../exists/indices/IndicesExistsRequest.java  |  4 +++-
 .../indices/IndicesExistsRequestBuilder.java  |  4 +++-
 .../exists/indices/IndicesExistsResponse.java |  4 +++-
 .../admin/indices/flush/FlushRequest.java     |  4 +++-
 .../indices/flush/FlushRequestBuilder.java    |  4 +++-
 .../admin/indices/flush/FlushResponse.java    |  4 +++-
 .../indices/forcemerge/ForceMergeRequest.java |  4 +++-
 .../forcemerge/ForceMergeRequestBuilder.java  |  4 +++-
 .../forcemerge/ForceMergeResponse.java        |  4 +++-
 .../admin/indices/get/GetIndexRequest.java    |  7 ++++--
 .../indices/get/GetIndexRequestBuilder.java   |  4 +++-
 .../admin/indices/get/GetIndexResponse.java   |  4 +++-
 .../mapping/get/GetFieldMappingsRequest.java  |  4 +++-
 .../get/GetFieldMappingsRequestBuilder.java   |  6 +++--
 .../mapping/get/GetFieldMappingsResponse.java |  7 ++++--
 .../mapping/get/GetMappingsRequest.java       |  4 +++-
 .../get/GetMappingsRequestBuilder.java        |  4 +++-
 .../mapping/get/GetMappingsResponse.java      |  4 +++-
 .../mapping/put/PutMappingRequest.java        |  4 +++-
 .../mapping/put/PutMappingRequestBuilder.java |  4 +++-
 .../admin/indices/open/OpenIndexRequest.java  |  4 +++-
 .../indices/open/OpenIndexRequestBuilder.java |  4 +++-
 .../admin/indices/open/OpenIndexResponse.java |  4 +++-
 .../readonly/AddIndexBlockRequest.java        |  4 +++-
 .../readonly/AddIndexBlockRequestBuilder.java |  4 +++-
 .../readonly/AddIndexBlockResponse.java       | 13 +++++++----
 .../indices/recovery/RecoveryRequest.java     |  4 +++-
 .../recovery/RecoveryRequestBuilder.java      |  4 +++-
 .../indices/recovery/RecoveryResponse.java    |  4 +++-
 .../admin/indices/refresh/RefreshRequest.java |  4 +++-
 .../refresh/RefreshRequestBuilder.java        |  4 +++-
 .../indices/refresh/RefreshResponse.java      |  4 +++-
 .../SegmentReplicationStatsRequest.java       |  4 +++-
 ...SegmentReplicationStatsRequestBuilder.java |  4 +++-
 .../SegmentReplicationStatsResponse.java      |  4 +++-
 .../indices/resolve/ResolveIndexAction.java   | 19 +++++++++++-----
 .../admin/indices/rollover/Condition.java     | 10 ++++++---
 .../indices/rollover/RolloverRequest.java     |  4 +++-
 .../rollover/RolloverRequestBuilder.java      |  4 +++-
 .../indices/rollover/RolloverResponse.java    |  4 +++-
 .../admin/indices/segments/IndexSegments.java |  5 ++++-
 .../indices/segments/IndexShardSegments.java  |  4 +++-
 .../segments/IndicesSegmentResponse.java      |  4 +++-
 .../segments/IndicesSegmentsRequest.java      |  4 +++-
 .../IndicesSegmentsRequestBuilder.java        |  4 +++-
 .../indices/segments/PitSegmentsRequest.java  |  4 ++++
 .../admin/indices/segments/ShardSegments.java |  4 +++-
 .../settings/get/GetSettingsRequest.java      |  4 +++-
 .../get/GetSettingsRequestBuilder.java        |  4 +++-
 .../settings/get/GetSettingsResponse.java     |  4 +++-
 .../settings/put/UpdateSettingsRequest.java   |  4 +++-
 .../put/UpdateSettingsRequestBuilder.java     |  4 +++-
 .../IndicesShardStoreRequestBuilder.java      |  4 +++-
 .../shards/IndicesShardStoresRequest.java     |  4 +++-
 .../shards/IndicesShardStoresResponse.java    | 13 ++++++++---
 .../admin/indices/shrink/ResizeRequest.java   |  4 +++-
 .../indices/shrink/ResizeRequestBuilder.java  |  4 +++-
 .../admin/indices/shrink/ResizeResponse.java  |  4 +++-
 .../admin/indices/shrink/ResizeType.java      |  5 ++++-
 .../admin/indices/stats/CommonStats.java      |  4 +++-
 .../admin/indices/stats/CommonStatsFlags.java |  7 ++++--
 .../admin/indices/stats/IndexShardStats.java  |  4 +++-
 .../admin/indices/stats/IndexStats.java       |  5 ++++-
 .../indices/stats/IndicesStatsRequest.java    |  4 +++-
 .../stats/IndicesStatsRequestBuilder.java     |  4 +++-
 .../indices/stats/IndicesStatsResponse.java   |  4 +++-
 .../admin/indices/stats/ShardStats.java       |  4 +++-
 .../delete/DeleteIndexTemplateRequest.java    |  4 +++-
 .../DeleteIndexTemplateRequestBuilder.java    |  4 +++-
 .../get/GetIndexTemplatesRequest.java         |  4 +++-
 .../get/GetIndexTemplatesRequestBuilder.java  |  4 +++-
 .../get/GetIndexTemplatesResponse.java        |  4 +++-
 .../post/SimulateIndexTemplateRequest.java    |  4 +++-
 .../post/SimulateIndexTemplateResponse.java   |  4 +++-
 .../put/PutComposableIndexTemplateAction.java |  7 ++++--
 .../template/put/PutIndexTemplateRequest.java |  4 +++-
 .../put/PutIndexTemplateRequestBuilder.java   |  4 +++-
 .../upgrade/get/IndexShardUpgradeStatus.java  |  4 +++-
 .../upgrade/get/IndexUpgradeStatus.java       |  5 ++++-
 .../upgrade/get/ShardUpgradeStatus.java       |  4 +++-
 .../upgrade/get/UpgradeStatusRequest.java     |  4 +++-
 .../get/UpgradeStatusRequestBuilder.java      |  4 +++-
 .../upgrade/get/UpgradeStatusResponse.java    |  4 +++-
 .../indices/upgrade/post/UpgradeRequest.java  |  4 +++-
 .../upgrade/post/UpgradeRequestBuilder.java   |  4 +++-
 .../indices/upgrade/post/UpgradeResponse.java |  4 +++-
 .../validate/query/QueryExplanation.java      |  4 +++-
 .../validate/query/ValidateQueryRequest.java  |  4 +++-
 .../query/ValidateQueryRequestBuilder.java    |  4 +++-
 .../validate/query/ValidateQueryResponse.java |  4 +++-
 .../action/bulk/BulkItemResponse.java         |  7 ++++--
 .../opensearch/action/bulk/BulkRequest.java   |  4 +++-
 .../action/bulk/BulkRequestBuilder.java       |  4 +++-
 .../opensearch/action/bulk/BulkResponse.java  |  4 +++-
 .../action/delete/DeleteRequest.java          |  4 +++-
 .../action/delete/DeleteRequestBuilder.java   |  4 +++-
 .../action/delete/DeleteResponse.java         |  7 ++++--
 .../action/explain/ExplainRequest.java        |  4 +++-
 .../action/explain/ExplainRequestBuilder.java |  4 +++-
 .../action/explain/ExplainResponse.java       |  4 +++-
 .../action/fieldcaps/FieldCapabilities.java   |  4 +++-
 .../fieldcaps/FieldCapabilitiesRequest.java   |  4 ++++
 .../FieldCapabilitiesRequestBuilder.java      |  4 +++-
 .../fieldcaps/FieldCapabilitiesResponse.java  |  5 ++++-
 .../org/opensearch/action/get/GetRequest.java |  4 +++-
 .../action/get/GetRequestBuilder.java         |  4 +++-
 .../opensearch/action/get/GetResponse.java    |  4 +++-
 .../action/get/MultiGetItemResponse.java      |  4 +++-
 .../action/get/MultiGetRequest.java           |  7 ++++--
 .../action/get/MultiGetRequestBuilder.java    |  4 +++-
 .../action/get/MultiGetResponse.java          |  7 ++++--
 .../opensearch/action/index/IndexRequest.java |  4 +++-
 .../action/index/IndexRequestBuilder.java     |  4 +++-
 .../action/index/IndexResponse.java           |  7 ++++--
 .../action/ingest/DeletePipelineRequest.java  |  4 +++-
 .../ingest/DeletePipelineRequestBuilder.java  |  4 +++-
 .../action/ingest/GetPipelineRequest.java     |  4 +++-
 .../ingest/GetPipelineRequestBuilder.java     |  4 +++-
 .../action/ingest/GetPipelineResponse.java    |  4 +++-
 .../action/ingest/PutPipelineRequest.java     |  4 +++-
 .../ingest/PutPipelineRequestBuilder.java     |  4 +++-
 .../action/ingest/SimulateDocumentResult.java |  4 +++-
 .../ingest/SimulatePipelineRequest.java       |  4 +++-
 .../SimulatePipelineRequestBuilder.java       |  4 +++-
 .../ingest/SimulatePipelineResponse.java      |  4 +++-
 .../action/search/ClearScrollRequest.java     |  4 +++-
 .../search/ClearScrollRequestBuilder.java     |  4 +++-
 .../action/search/ClearScrollResponse.java    |  4 +++-
 .../action/search/CreatePitRequest.java       |  4 ++++
 .../action/search/CreatePitResponse.java      |  4 ++++
 .../action/search/DeletePitInfo.java          |  4 ++++
 .../action/search/DeletePitRequest.java       |  4 ++++
 .../action/search/DeletePitResponse.java      |  4 ++++
 .../search/DeleteSearchPipelineRequest.java   |  4 +++-
 .../action/search/GetAllPitNodeResponse.java  |  4 ++++
 .../action/search/GetAllPitNodesRequest.java  |  4 ++++
 .../action/search/GetAllPitNodesResponse.java |  4 ++++
 .../search/GetSearchPipelineRequest.java      |  4 +++-
 .../search/GetSearchPipelineResponse.java     |  4 +++-
 .../opensearch/action/search/ListPitInfo.java |  4 ++++
 .../action/search/MultiSearchRequest.java     |  4 +++-
 .../search/MultiSearchRequestBuilder.java     |  4 +++-
 .../action/search/MultiSearchResponse.java    |  7 ++++--
 .../search/PutSearchPipelineRequest.java      |  4 +++-
 .../action/search/SearchPhaseName.java        |  6 ++++-
 .../action/search/SearchProgressListener.java |  2 ++
 .../action/search/SearchRequest.java          |  4 +++-
 .../action/search/SearchRequestBuilder.java   |  4 +++-
 .../action/search/SearchRequestStats.java     |  4 +++-
 .../action/search/SearchResponse.java         | 10 ++++++---
 .../action/search/SearchResponseSections.java |  4 +++-
 .../action/search/SearchScrollRequest.java    |  4 +++-
 .../search/SearchScrollRequestBuilder.java    |  4 +++-
 .../opensearch/action/search/SearchTask.java  |  4 +++-
 .../opensearch/action/search/SearchType.java  |  5 ++++-
 .../action/support/IndicesOptions.java        |  7 ++++--
 .../action/support/WriteRequest.java          |  4 +++-
 .../support/master/AcknowledgedResponse.java  |  4 +++-
 .../MultiTermVectorsItemResponse.java         |  4 +++-
 .../termvectors/MultiTermVectorsRequest.java  |  4 +++-
 .../MultiTermVectorsRequestBuilder.java       |  4 +++-
 .../termvectors/MultiTermVectorsResponse.java |  7 ++++--
 .../action/termvectors/TermVectorsFilter.java |  7 ++++--
 .../termvectors/TermVectorsRequest.java       | 10 ++++++---
 .../TermVectorsRequestBuilder.java            |  4 +++-
 .../termvectors/TermVectorsResponse.java      |  4 +++-
 .../action/update/UpdateRequest.java          |  4 +++-
 .../action/update/UpdateRequestBuilder.java   |  4 +++-
 .../action/update/UpdateResponse.java         |  7 ++++--
 .../cluster/ClusterChangedEvent.java          |  4 +++-
 .../org/opensearch/cluster/ClusterInfo.java   |  7 ++++--
 .../org/opensearch/cluster/ClusterName.java   |  4 +++-
 .../java/org/opensearch/cluster/Diff.java     |  4 +++-
 .../org/opensearch/cluster/DiskUsage.java     |  4 +++-
 .../ClusterAwarenessAttributeValueHealth.java |  4 ++++
 .../ClusterAwarenessAttributesHealth.java     |  3 +++
 .../ClusterAwarenessHealth.java               |  4 ++++
 .../cluster/block/ClusterBlock.java           |  4 +++-
 .../cluster/block/ClusterBlockLevel.java      |  5 ++++-
 .../decommission/DecommissionAttribute.java   |  4 +++-
 .../decommission/DecommissionStatus.java      |  5 +++++
 .../cluster/health/ClusterHealthStatus.java   |  4 +++-
 .../cluster/health/ClusterIndexHealth.java    |  4 +++-
 .../cluster/health/ClusterShardHealth.java    |  4 +++-
 .../cluster/metadata/AliasMetadata.java       |  7 ++++--
 .../cluster/metadata/CryptoMetadata.java      |  4 +++-
 .../cluster/metadata/DataStream.java          |  7 ++++--
 .../cluster/metadata/IndexMetadata.java       |  4 +++-
 .../metadata/IndexTemplateMetadata.java       |  3 ++-
 .../cluster/metadata/MappingMetadata.java     |  4 +++-
 .../cluster/metadata/RepositoryMetadata.java  |  4 +++-
 .../cluster/node/DiscoveryNodes.java          | 10 ++++++---
 .../cluster/routing/RecoverySource.java       |  7 ++++--
 .../cluster/routing/ShardRoutingState.java    |  5 ++++-
 .../cluster/routing/UnassignedInfo.java       | 10 ++++++---
 .../cluster/routing/WeightedRouting.java      |  4 +++-
 .../allocation/RerouteExplanation.java        |  4 +++-
 .../allocation/RoutingExplanations.java       |  4 +++-
 .../routing/allocation/decider/Decision.java  |  7 ++++--
 .../service/ClusterManagerTaskThrottler.java  |  4 ++++
 .../cluster/service/ClusterService.java       |  4 +---
 .../cluster/service/PendingClusterTask.java   |  4 +++-
 .../opensearch/common/FieldMemoryStats.java   |  4 +++-
 .../java/org/opensearch/common/Priority.java  |  4 +++-
 .../blobstore/stream/read/ReadContext.java    |  5 +++++
 .../common/document/DocumentField.java        |  4 +++-
 .../opensearch/common/geo/ShapeRelation.java  |  4 +++-
 .../org/opensearch/common/inject/Module.java  |  5 +----
 .../common/io/stream/BytesStreamOutput.java   |  4 +++-
 .../common/logging/DeprecationLogger.java     |  7 ++++--
 .../common/settings/IndexScopedSettings.java  |  4 +++-
 .../org/opensearch/common/unit/Fuzziness.java |  4 +++-
 .../org/opensearch/common/unit/SizeValue.java |  4 +++-
 .../java/org/opensearch/http/HttpChannel.java |  4 +++-
 .../java/org/opensearch/http/HttpRequest.java |  7 ++++--
 .../org/opensearch/http/HttpResponse.java     |  5 ++++-
 .../org/opensearch/index/IndexModule.java     |  3 ++-
 .../org/opensearch/index/IndexSortConfig.java |  7 ++++--
 .../index/MergeSchedulerConfig.java           |  4 +++-
 .../opensearch/index/ReplicationStats.java    |  4 +++-
 .../SegmentReplicationPerGroupStats.java      |  4 +++-
 .../index/SegmentReplicationShardStats.java   |  4 +++-
 .../org/opensearch/index/VersionType.java     |  4 +++-
 .../index/analysis/AnalysisMode.java          |  5 ++++-
 .../index/analysis/AnalyzerScope.java         |  5 ++++-
 .../index/analysis/NameOrDefinition.java      |  4 +++-
 .../index/analysis/NamedAnalyzer.java         |  4 +++-
 .../index/cache/query/QueryCacheStats.java    |  4 +++-
 .../cache/request/RequestCacheStats.java      |  4 +++-
 .../opensearch/index/engine/CommitStats.java  |  4 +++-
 .../org/opensearch/index/engine/Segment.java  |  4 +++-
 .../index/engine/SegmentsStats.java           |  4 +++-
 .../index/fielddata/FieldDataStats.java       |  4 +++-
 .../index/fielddata/ScriptDocValues.java      |  4 +++-
 .../fielddata/SortedBinaryDocValues.java      |  4 +++-
 .../opensearch/index/flush/FlushStats.java    |  4 +++-
 .../org/opensearch/index/get/GetResult.java   |  2 ++
 .../org/opensearch/index/get/GetStats.java    |  2 ++
 .../opensearch/index/merge/MergeStats.java    |  4 +++-
 .../opensearch/index/query/IntervalMode.java  |  4 +++-
 .../opensearch/index/query/QueryBuilder.java  |  4 +++-
 .../index/query/QueryBuilderVisitor.java      |  4 ++++
 .../index/query/QueryRewriteContext.java      |  4 +++-
 .../index/recovery/RecoveryStats.java         |  4 +++-
 .../index/refresh/RefreshStats.java           |  4 +++-
 .../index/remote/RemoteSegmentStats.java      |  4 +++-
 .../remote/RemoteSegmentTransferTracker.java  |  7 ++++--
 .../remote/RemoteTranslogTransferTracker.java |  7 ++++--
 .../index/search/stats/SearchStats.java       | 14 +++++++-----
 .../index/seqno/RetentionLease.java           |  4 +++-
 .../index/seqno/RetentionLeaseStats.java      |  4 +++-
 .../index/seqno/RetentionLeases.java          |  4 +++-
 .../opensearch/index/seqno/SeqNoStats.java    |  4 +++-
 .../org/opensearch/index/shard/DocsStats.java |  4 +++-
 .../opensearch/index/shard/IndexingStats.java | 10 ++++++---
 .../store/DirectoryFileTransferTracker.java   |  7 ++++--
 .../opensearch/index/store/StoreStats.java    |  4 +++-
 .../remote/filecache/FileCacheStats.java      |  4 +++-
 .../index/translog/RemoteTranslogStats.java   |  4 +++-
 .../opensearch/index/translog/Translog.java   |  4 +++-
 .../index/translog/TranslogStats.java         |  4 +++-
 .../opensearch/index/warmer/WarmerStats.java  |  4 +++-
 .../indices/recovery/RecoverySettings.java    |  4 +++-
 .../indices/recovery/RecoveryState.java       | 13 +++++++----
 .../replication/SegmentReplicationState.java  |  7 ++++--
 .../common/ReplicationLuceneIndex.java        |  7 ++++--
 .../replication/common/ReplicationTimer.java  |  4 +++-
 .../ingest/PipelineConfiguration.java         |  4 +++-
 .../org/opensearch/monitor/fs/FsInfo.java     | 13 +++++++----
 .../org/opensearch/monitor/os/OsStats.java    | 19 +++++++++++-----
 .../plugins/CryptoKeyProviderPlugin.java      |  3 ++-
 .../org/opensearch/plugins/PluginInfo.java    |  2 ++
 .../repositories/RepositoriesStats.java       |  6 ++++-
 .../repositories/RepositoryCleanupResult.java |  4 +++-
 .../java/org/opensearch/rest/RestRequest.java |  5 ++++-
 .../opensearch/script/ScriptCacheStats.java   |  2 ++
 .../opensearch/script/ScriptContextInfo.java  |  7 ++++--
 .../opensearch/script/ScriptContextStats.java |  4 +++-
 .../script/ScriptLanguagesInfo.java           |  4 +++-
 .../org/opensearch/script/ScriptStats.java    |  4 +++-
 .../opensearch/script/StoredScriptSource.java |  4 +++-
 .../java/org/opensearch/search/Scroll.java    |  4 +++-
 .../opensearch/search/SearchExtBuilder.java   |  4 +++-
 .../search/builder/PointInTimeBuilder.java    |  4 +++-
 .../search/collapse/CollapseBuilder.java      |  4 +++-
 .../search/collapse/CollapseContext.java      |  4 +++-
 .../fetch/subphase/FetchSourceContext.java    |  4 +++-
 .../subphase/highlight/HighlightBuilder.java  | 13 +++++++----
 .../highlight/SearchHighlightContext.java     | 10 ++++++---
 .../search/internal/AliasFilter.java          |  4 +++-
 .../pipeline/PipelineConfiguration.java       |  4 ++++
 .../search/profile/NetworkTime.java           |  4 +++-
 .../search/profile/ProfileResult.java         |  4 +++-
 .../search/profile/ProfileShardResult.java    |  4 +++-
 .../AggregationProfileShardResult.java        |  4 +++-
 .../search/profile/query/CollectorResult.java |  4 +++-
 .../query/QueryProfileShardResult.java        |  4 +++-
 .../opensearch/search/slice/SliceBuilder.java |  8 +++----
 .../org/opensearch/search/sort/SortOrder.java |  4 +++-
 .../opensearch/search/suggest/Suggest.java    | 12 +++++++++-
 .../search/suggest/SuggestBuilder.java        |  4 +++-
 .../opensearch/search/suggest/Suggester.java  |  4 +++-
 .../search/suggest/SuggestionBuilder.java     |  4 +++-
 .../suggest/SuggestionSearchContext.java      |  7 ++++--
 .../suggest/completion/CompletionStats.java   |  4 +++-
 .../org/opensearch/snapshots/RestoreInfo.java |  4 +++-
 .../org/opensearch/snapshots/Snapshot.java    |  4 +++-
 .../org/opensearch/snapshots/SnapshotId.java  |  4 +++-
 .../opensearch/snapshots/SnapshotInfo.java    |  4 +++-
 .../snapshots/SnapshotShardFailure.java       |  4 +++-
 .../opensearch/snapshots/SnapshotState.java   |  5 ++++-
 .../main/java/org/opensearch/tasks/Task.java  | 12 +++++-----
 .../java/org/opensearch/tasks/TaskInfo.java   |  4 +++-
 .../java/org/opensearch/tasks/TaskResult.java |  4 +++-
 .../threadpool/ExecutorBuilder.java           |  4 +++-
 .../org/opensearch/threadpool/Scheduler.java  |  7 ++++++
 .../org/opensearch/threadpool/ThreadPool.java |  6 +++--
 .../opensearch/threadpool/ThreadPoolInfo.java |  4 +++-
 .../threadpool/ThreadPoolStats.java           |  7 ++++--
 .../opensearch/watcher/ResourceWatcher.java   |  5 ++++-
 .../watcher/ResourceWatcherService.java       |  3 ++-
 .../org/opensearch/watcher/WatcherHandle.java |  5 ++++-
 497 files changed, 1757 insertions(+), 579 deletions(-)

diff --git a/libs/common/src/main/java/org/opensearch/common/CheckedBiConsumer.java b/libs/common/src/main/java/org/opensearch/common/CheckedBiConsumer.java
index 50c15bb7a95a8..c2ef08e288346 100644
--- a/libs/common/src/main/java/org/opensearch/common/CheckedBiConsumer.java
+++ b/libs/common/src/main/java/org/opensearch/common/CheckedBiConsumer.java
@@ -32,13 +32,16 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.function.BiConsumer;
 
 /**
  * A {@link BiConsumer}-like interface which allows throwing checked exceptions.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface CheckedBiConsumer {
     void accept(T t, U u) throws E;
diff --git a/libs/common/src/main/java/org/opensearch/common/CheckedFunction.java b/libs/common/src/main/java/org/opensearch/common/CheckedFunction.java
index 9c17ad4b4ee3f..927edd1b9905a 100644
--- a/libs/common/src/main/java/org/opensearch/common/CheckedFunction.java
+++ b/libs/common/src/main/java/org/opensearch/common/CheckedFunction.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.function.Function;
 
 /**
@@ -39,6 +41,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface CheckedFunction {
     R apply(T t) throws E;
diff --git a/libs/common/src/main/java/org/opensearch/common/TriFunction.java b/libs/common/src/main/java/org/opensearch/common/TriFunction.java
index 7b1bbece68680..8594e8e2cd0c9 100644
--- a/libs/common/src/main/java/org/opensearch/common/TriFunction.java
+++ b/libs/common/src/main/java/org/opensearch/common/TriFunction.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Represents a function that accepts three arguments and produces a result.
  *
@@ -40,8 +42,9 @@
  * @param  the type of the third argument
  * @param  the return type
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface TriFunction {
     /**
diff --git a/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java b/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java
index 5c0e3f2de7708..a5d97dcd85ef7 100644
--- a/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java
+++ b/libs/common/src/main/java/org/opensearch/common/collect/Tuple.java
@@ -32,12 +32,15 @@
 
 package org.opensearch.common.collect;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Java 9 Tuple
  * todo: deprecate and remove w/ min jdk upgrade to 11?
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Tuple {
 
     public static  Tuple tuple(V1 v1, V2 v2) {
diff --git a/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java b/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java
index 8afa48eb92c0f..31d2dcd0dba3d 100644
--- a/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java
+++ b/libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java
@@ -7,12 +7,17 @@
  */
 package org.opensearch.common.crypto;
 
+import org.opensearch.common.annotation.ExperimentalApi;
+
 import java.io.Closeable;
 import java.util.Map;
 
 /**
  * Master key provider responsible for management of master keys.
+ *
+ * @opensearch.experimental
  */
+@ExperimentalApi
 public interface MasterKeyProvider extends Closeable {
 
     /**
diff --git a/libs/common/src/main/java/org/opensearch/common/io/InputStreamContainer.java b/libs/common/src/main/java/org/opensearch/common/io/InputStreamContainer.java
index eb8a4e1382497..3095336338f7f 100644
--- a/libs/common/src/main/java/org/opensearch/common/io/InputStreamContainer.java
+++ b/libs/common/src/main/java/org/opensearch/common/io/InputStreamContainer.java
@@ -8,13 +8,16 @@
 
 package org.opensearch.common.io;
 
+import org.opensearch.common.annotation.ExperimentalApi;
+
 import java.io.InputStream;
 
 /**
  * Model composed of an input stream and the total content length of the stream
  *
- * @opensearch.internal
+ * @opensearch.experimental
  */
+@ExperimentalApi
 public class InputStreamContainer {
 
     private final InputStream inputStream;
diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchParseException.java b/libs/core/src/main/java/org/opensearch/OpenSearchParseException.java
index c2516402b0d30..26aff04b30a56 100644
--- a/libs/core/src/main/java/org/opensearch/OpenSearchParseException.java
+++ b/libs/core/src/main/java/org/opensearch/OpenSearchParseException.java
@@ -32,6 +32,7 @@
 
 package org.opensearch;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.rest.RestStatus;
 
@@ -40,8 +41,9 @@
 /**
  * Unchecked exception that is translated into a {@code 400 BAD REQUEST} error when it bubbles out over HTTP.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenSearchParseException extends OpenSearchException {
 
     public OpenSearchParseException(String msg, Object... args) {
diff --git a/libs/core/src/main/java/org/opensearch/core/action/NotifyOnceListener.java b/libs/core/src/main/java/org/opensearch/core/action/NotifyOnceListener.java
index 6af9ca005d171..f087322e0024c 100644
--- a/libs/core/src/main/java/org/opensearch/core/action/NotifyOnceListener.java
+++ b/libs/core/src/main/java/org/opensearch/core/action/NotifyOnceListener.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.core.action;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -39,8 +41,9 @@
  * the is called is only called once. Subclasses should implement notification logic with
  * innerOnResponse and innerOnFailure.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class NotifyOnceListener implements ActionListener {
 
     private final AtomicBoolean hasBeenCalled = new AtomicBoolean(false);
diff --git a/libs/core/src/main/java/org/opensearch/core/common/text/Text.java b/libs/core/src/main/java/org/opensearch/core/common/text/Text.java
index ca5402edae59e..3a46bd4602297 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/text/Text.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/text/Text.java
@@ -32,6 +32,7 @@
 package org.opensearch.core.common.text;
 
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.ToXContentFragment;
@@ -44,8 +45,9 @@
  * Both {@link String} and {@link BytesReference} representation of the text. Starts with one of those, and if
  * the other is requests, caches the other one in a local reference so no additional conversion will be needed.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class Text implements Comparable, ToXContentFragment {
 
     public static final Text[] EMPTY_ARRAY = new Text[0];
diff --git a/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java b/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java
index 551504ed6f719..3b5fbb7d76307 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.common.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.network.NetworkAddress;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -47,8 +48,9 @@
 /**
  * A transport address used for IP socket address (wraps {@link java.net.InetSocketAddress}).
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class TransportAddress implements Writeable, ToXContentFragment {
 
     /**
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStats.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStats.java
index d65f75581dd1b..e99afbb759031 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStats.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStats.java
@@ -8,11 +8,14 @@
 
 package org.opensearch.core.tasks.resourcetracker;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  *  Different resource stats are defined.
  *
- *  @opensearch.internal
+ *  @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public enum ResourceStats {
     CPU("cpu_time_in_nanos"),
     MEMORY("memory_in_bytes");
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStatsType.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStatsType.java
index fce8cc65e9bc5..2aedff2940d83 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStatsType.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceStatsType.java
@@ -8,11 +8,14 @@
 
 package org.opensearch.core.tasks.resourcetracker;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Defines the different types of resource stats.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public enum ResourceStatsType {
     // resource stats of the worker thread reported directly from runnable.
     WORKER_STATS("worker_stats", false);
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageInfo.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageInfo.java
index 2cbc3d4b2f5c3..a278b61894a65 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageInfo.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageInfo.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.Collections;
 import java.util.EnumMap;
@@ -22,8 +23,9 @@
  * It captures the resource usage information like memory, CPU about a particular execution of thread
  * for a specific stats type.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public class ResourceUsageInfo {
     private static final Logger logger = LogManager.getLogger(ResourceUsageInfo.class);
     private final EnumMap statsInfo = new EnumMap<>(ResourceStats.class);
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageMetric.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageMetric.java
index 262dbe20dabda..f4cce2de820a0 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageMetric.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageMetric.java
@@ -8,11 +8,14 @@
 
 package org.opensearch.core.tasks.resourcetracker;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Information about resource usage
  *
- * @opensearch.internal
+ *  @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public class ResourceUsageMetric {
     private final ResourceStats stats;
     private final long value;
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceStats.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceStats.java
index d0d26550a4742..048c4a228fbd5 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceStats.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceStats.java
@@ -9,6 +9,7 @@
 package org.opensearch.core.tasks.resourcetracker;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -29,8 +30,9 @@
  * Writeable TaskResourceStats objects are used to represent resource
  * snapshot information about currently running task.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public class TaskResourceStats implements Writeable, ToXContentFragment {
     private final Map resourceUsage;
     private final TaskThreadUsage threadUsage;
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceUsage.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceUsage.java
index 7d6cadbef23d7..654f1c5695937 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceUsage.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskResourceUsage.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.core.tasks.resourcetracker;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -30,8 +31,9 @@
  * Writeable TaskResourceUsage objects are used to represent resource usage
  * information of running tasks.
  *
- * @opensearch.internal
+ *  @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public class TaskResourceUsage implements Writeable, ToXContentFragment {
     private static final ParseField CPU_TIME_IN_NANOS = new ParseField("cpu_time_in_nanos");
     private static final ParseField MEMORY_IN_BYTES = new ParseField("memory_in_bytes");
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskThreadUsage.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskThreadUsage.java
index b593ec96e5996..abe03e3c520e0 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskThreadUsage.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/TaskThreadUsage.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.core.tasks.resourcetracker;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -29,8 +30,9 @@
  * Writeable TaskThreadExecutions objects are used to represent thread related resource usage of running tasks.
  * asd
  *
- * @opensearch.internal
+ *  @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public class TaskThreadUsage implements Writeable, ToXContentFragment {
 
     private static final String THREAD_EXECUTIONS = "thread_executions";
diff --git a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ThreadResourceInfo.java b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ThreadResourceInfo.java
index 4b341a94256c4..703fdfdf8a784 100644
--- a/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ThreadResourceInfo.java
+++ b/libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ThreadResourceInfo.java
@@ -8,14 +8,17 @@
 
 package org.opensearch.core.tasks.resourcetracker;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Resource consumption information about a particular execution of thread.
  * 
                              
  * It captures the resource usage information about a particular execution of thread
  * for a specific stats type like worker_stats or response_stats etc.,
  *
- * @opensearch.internal
+ *  @opensearch.api
  */
+@PublicApi(since = "2.1.0")
 public class ThreadResourceInfo {
     private final long threadId;
     private volatile boolean isActive = true;
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java
index 79e531a542026..32bbfc600f1f0 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/AbstractObjectParser.java
@@ -33,6 +33,7 @@
 package org.opensearch.core.xcontent;
 
 import org.opensearch.common.CheckedFunction;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.xcontent.ObjectParser.NamedObjectParser;
 import org.opensearch.core.xcontent.ObjectParser.ValueType;
@@ -47,8 +48,9 @@
 /**
  * Superclass for {@link ObjectParser} and {@link ConstructingObjectParser}. Defines most of the "declare" methods so they can be shared.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class AbstractObjectParser {
 
     /**
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/ContextParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/ContextParser.java
index d50dd2e68d890..f6e5647532bee 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/ContextParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/ContextParser.java
@@ -32,11 +32,16 @@
 
 package org.opensearch.core.xcontent;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.IOException;
 
 /**
  * Reads an object from a parser using some context.
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface ContextParser {
     T parse(XContentParser p, Context c) throws IOException;
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/DeprecationHandler.java b/libs/core/src/main/java/org/opensearch/core/xcontent/DeprecationHandler.java
index 570a13ad8e093..a0e4027290742 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/DeprecationHandler.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/DeprecationHandler.java
@@ -32,12 +32,17 @@
 
 package org.opensearch.core.xcontent;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.function.Supplier;
 
 /**
  * Callback for notifying the creator of the {@link XContentParser} that
  * parsing hit a deprecated field.
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface DeprecationHandler {
     /**
      * Throws an {@link UnsupportedOperationException} when parsing hits a
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/ToXContent.java b/libs/core/src/main/java/org/opensearch/core/xcontent/ToXContent.java
index 90dd0cbfb9a1a..ee8dad198df09 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/ToXContent.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/ToXContent.java
@@ -33,6 +33,7 @@
 package org.opensearch.core.xcontent;
 
 import org.opensearch.common.Booleans;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.io.IOException;
 import java.util.Map;
@@ -42,15 +43,17 @@
  * The output may or may not be a value object. Objects implementing {@link ToXContentObject} output a valid value
  * but those that don't may or may not require emitting a startObject and an endObject.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ToXContent {
 
     /**
      * Base parameters class
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     interface Params {
         String param(String key);
 
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContent.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContent.java
index dbc0041af42b5..1ebdd69d2b7a3 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContent.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContent.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.core.xcontent;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -41,7 +43,10 @@
 
 /**
  * A generic abstraction on top of handling content, inspired by JSON and pull parsing.
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface XContent {
     /**
      * The type this content handles and produces.
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java
index 4bfd47ccfdc94..85c3579b74cd5 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParser.java
@@ -33,6 +33,7 @@
 package org.opensearch.core.xcontent;
 
 import org.opensearch.common.CheckedFunction;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -53,8 +54,9 @@
  *          NamedXContentRegistry.EMPTY, ParserField."{\"key\" : \"value\"}");
  * 
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface XContentParser extends Closeable {
 
     /**
diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java
index 9d1581a3a1517..453107fe4ff65 100644
--- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java
@@ -35,6 +35,7 @@
 import com.fasterxml.jackson.dataformat.cbor.CBORConstants;
 import com.fasterxml.jackson.dataformat.smile.SmileConstants;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.cbor.CborXContent;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.common.xcontent.smile.SmileXContent;
@@ -49,7 +50,9 @@
 
 /**
  * The content type of {@link XContent}.
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum XContentType implements MediaType {
 
     /**
diff --git a/server/src/main/java/org/opensearch/OpenSearchGenerationException.java b/server/src/main/java/org/opensearch/OpenSearchGenerationException.java
index 50b021d41123f..51093390d57e0 100644
--- a/server/src/main/java/org/opensearch/OpenSearchGenerationException.java
+++ b/server/src/main/java/org/opensearch/OpenSearchGenerationException.java
@@ -32,6 +32,7 @@
 
 package org.opensearch;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -39,8 +40,9 @@
 /**
  * A generic exception indicating failure to generate.
  *
- * @opensearch.internal
+  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenSearchGenerationException extends OpenSearchException {
 
     public OpenSearchGenerationException(String msg) {
diff --git a/server/src/main/java/org/opensearch/action/ActionRequestValidationException.java b/server/src/main/java/org/opensearch/action/ActionRequestValidationException.java
index ffba4d2eb50c0..c30da41066dbd 100644
--- a/server/src/main/java/org/opensearch/action/ActionRequestValidationException.java
+++ b/server/src/main/java/org/opensearch/action/ActionRequestValidationException.java
@@ -33,10 +33,12 @@
 package org.opensearch.action;
 
 import org.opensearch.common.ValidationException;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Base exception for an action request validation extendable by plugins
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ActionRequestValidationException extends ValidationException {}
diff --git a/server/src/main/java/org/opensearch/action/DocWriteRequest.java b/server/src/main/java/org/opensearch/action/DocWriteRequest.java
index 31f61e76c74ff..e09c76430defc 100644
--- a/server/src/main/java/org/opensearch/action/DocWriteRequest.java
+++ b/server/src/main/java/org/opensearch/action/DocWriteRequest.java
@@ -38,6 +38,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -55,8 +56,9 @@
  * Generic interface to group ActionRequest, which perform writes to a single document
  * Action requests implementing this can be part of {@link org.opensearch.action.bulk.BulkRequest}
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface DocWriteRequest extends IndicesRequest, Accountable {
 
     // Flag set for disallowing index auto creation for an individual write request.
@@ -169,7 +171,10 @@ public interface DocWriteRequest extends IndicesRequest, Accountable {
 
     /**
      * Requested operation type to perform on the document
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     enum OpType {
         /**
          * Index the source. If there an existing document with the id, it will
diff --git a/server/src/main/java/org/opensearch/action/DocWriteResponse.java b/server/src/main/java/org/opensearch/action/DocWriteResponse.java
index e3052b3b80035..aada56ed93fd3 100644
--- a/server/src/main/java/org/opensearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/opensearch/action/DocWriteResponse.java
@@ -38,6 +38,7 @@
 import org.opensearch.action.support.replication.ReplicationResponse;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -64,8 +65,9 @@
 /**
  * A base class for the response of a write operation that involves a single doc
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContentObject {
 
     private static final String _SHARDS = "_shards";
@@ -80,7 +82,10 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
     /**
      * An enum that represents the results of CRUD operations, primarily used to communicate the type of
      * operation that occurred.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Result implements Writeable {
         CREATED(0),
         UPDATED(1),
diff --git a/server/src/main/java/org/opensearch/action/IndicesRequest.java b/server/src/main/java/org/opensearch/action/IndicesRequest.java
index 7e4c2f5076cda..f42818595d3cc 100644
--- a/server/src/main/java/org/opensearch/action/IndicesRequest.java
+++ b/server/src/main/java/org/opensearch/action/IndicesRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action;
 
 import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Needs to be implemented by all {@link org.opensearch.action.ActionRequest} subclasses that relate to
@@ -40,8 +41,9 @@
  * In case of internal requests originated during the distributed execution of an external request,
  * they will still return the indices that the original request related to.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface IndicesRequest {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
index 3dec781f0acf4..4a5c3b076faba 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -48,8 +49,9 @@
 /**
  * A request to explain the allocation of a shard in the cluster
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterAllocationExplainRequest extends ClusterManagerNodeRequest {
 
     private static final ObjectParser PARSER = new ObjectParser<>("cluster/allocation/explain");
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
index d85cb3929873d..31781dda04957 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for requests to explain the allocation of a shard in the cluster
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterAllocationExplainRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     ClusterAllocationExplainRequest,
     ClusterAllocationExplainResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
index 0eeedb9af0ab7..17afdd862cf66 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.allocation;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -41,8 +42,9 @@
 /**
  * Explanation response for a shard in the cluster
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterAllocationExplainResponse extends ActionResponse {
 
     private ClusterAllocationExplanation cae;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/crypto/CryptoSettings.java b/server/src/main/java/org/opensearch/action/admin/cluster/crypto/CryptoSettings.java
index bd783b349bed4..6e4d7b84f204e 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/crypto/CryptoSettings.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/crypto/CryptoSettings.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.crypto;
 
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -28,8 +29,9 @@
 /**
  * Crypto settings supplied during a put repository request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CryptoSettings implements Writeable, ToXContentObject {
     private String keyProviderName;
     private String keyProviderType;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java
index 79b7381801da6..8243be21ba487 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -18,8 +19,9 @@
 /**
  * Request for deleting decommission request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class DeleteDecommissionStateRequest extends ClusterManagerNodeRequest {
 
     public DeleteDecommissionStateRequest() {}
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequestBuilder.java
index 08f194c53f18e..94075d6ec860d 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateRequestBuilder.java
@@ -10,12 +10,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for Delete decommission request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class DeleteDecommissionStateRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     DeleteDecommissionStateRequest,
     DeleteDecommissionStateResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java
index 3d0404c25373b..13b056a019200 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.decommission.awareness.delete;
 
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -17,8 +18,9 @@
 /**
  * Response returned after deletion of decommission request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class DeleteDecommissionStateResponse extends AcknowledgedResponse {
 
     public DeleteDecommissionStateResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java
index 15c7e165fb62f..3ecbbfbbc7285 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -21,8 +22,9 @@
 /**
  * Get Decommissioned attribute request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class GetDecommissionStateRequest extends ClusterManagerNodeReadRequest {
 
     private String attributeName;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequestBuilder.java
index e766e9c674ff7..13eb375f0d00e 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateRequestBuilder.java
@@ -10,12 +10,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Get decommission request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class GetDecommissionStateRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     GetDecommissionStateRequest,
     GetDecommissionStateResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java
index bbcbd7013c299..9010c0e7d9388 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/get/GetDecommissionStateResponse.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.cluster.decommission.DecommissionStatus;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -26,8 +27,9 @@
 /**
  * Response for decommission status
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class GetDecommissionStateResponse extends ActionResponse implements ToXContentObject {
 
     private String attributeValue;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java
index 18f14db3c2b8d..9070aa5a0dc55 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequest.java
@@ -11,6 +11,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.cluster.decommission.DecommissionAttribute;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -23,8 +24,9 @@
 /**
  * Registers a decommission request with decommission attribute and timeout
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class DecommissionRequest extends ClusterManagerNodeRequest {
 
     public static final TimeValue DEFAULT_NODE_DRAINING_TIMEOUT = TimeValue.timeValueSeconds(120);
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequestBuilder.java
index c3591fff54885..e965110cdb9df 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionRequestBuilder.java
@@ -12,13 +12,15 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.decommission.DecommissionAttribute;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
  * Register decommission request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class DecommissionRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     DecommissionRequest,
     DecommissionResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java
index 13c1775b005b3..a2401cdf91b07 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.decommission.awareness.put;
 
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentObject;
@@ -18,8 +19,9 @@
 /**
  * Response for decommission request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class DecommissionResponse extends AcknowledgedResponse implements ToXContentObject {
 
     public DecommissionResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java
index a9532077f9129..ec8b01d853da6 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java
@@ -40,6 +40,7 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.common.Priority;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -53,8 +54,9 @@
 /**
  * Transport request for requesting cluster health
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterHealthRequest extends ClusterManagerNodeReadRequest implements IndicesRequest.Replaceable {
 
     private String[] indices;
@@ -351,8 +353,9 @@ public ActionRequestValidationException validate() {
     /**
      * The level of the health request.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Level {
         CLUSTER,
         INDICES,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java
index cca9d35d8aa6f..a9a3756755265 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java
@@ -38,13 +38,15 @@
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.common.Priority;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
  * Builder for requesting cluster health
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterHealthRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     ClusterHealthRequest,
     ClusterHealthResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java
index fb68012502116..1a27f161343e8 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java
@@ -38,6 +38,7 @@
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.cluster.health.ClusterIndexHealth;
 import org.opensearch.cluster.health.ClusterStateHealth;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.StatusToXContentObject;
@@ -67,8 +68,9 @@
 /**
  * Transport response for Cluster Health
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterHealthResponse extends ActionResponse implements StatusToXContentObject {
     private static final String CLUSTER_NAME = "cluster_name";
     private static final String STATUS = "status";
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
index 39e912a995f35..9e52b90f7bd38 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.node.hotthreads;
 
 import org.opensearch.action.support.nodes.BaseNodesRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -43,8 +44,9 @@
 /**
  * Transport request for OpenSearch Hot Threads
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesHotThreadsRequest extends BaseNodesRequest {
 
     int threads = 3;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java
index 3639439dd3fb8..51b455b41115b 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
  * Builder class for requesting OpenSearch Hot Threads
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesHotThreadsRequestBuilder extends NodesOperationRequestBuilder<
     NodesHotThreadsRequest,
     NodesHotThreadsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java
index 5af9ce50a4bfe..eeddd2deb7ff8 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
 import org.opensearch.cluster.ClusterName;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -44,8 +45,9 @@
 /**
  * Transport response for OpenSearch Hot Threads
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesHotThreadsResponse extends BaseNodesResponse {
 
     public NodesHotThreadsResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java
index 31cacda7c3487..e694a5e102e02 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.node.info;
 
 import org.opensearch.action.support.nodes.BaseNodesRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -47,8 +48,9 @@
 /**
  * A request to get node (cluster) level information.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesInfoRequest extends BaseNodesRequest {
 
     private Set requestedMetrics = Metric.allMetrics();
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
index 76ef75b77a1cf..4c3191b017948 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport action for OpenSearch Node Information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesInfoRequestBuilder extends NodesOperationRequestBuilder {
 
     public NodesInfoRequestBuilder(OpenSearchClient client, NodesInfoAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java
index 5b4444053a8b7..7ddd70185e8ad 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoResponse.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.nodes.BaseNodesResponse;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -59,8 +60,9 @@
 /**
  * Transport response for OpenSearch Node Information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesInfoResponse extends BaseNodesResponse implements ToXContentFragment {
 
     public NodesInfoResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java
index 2f2162947aeea..d970e4c9e5468 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.settings.SecureString;
 
 /**
  * Builder for the reload secure settings nodes request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesReloadSecureSettingsRequestBuilder extends NodesOperationRequestBuilder<
     NodesReloadSecureSettingsRequest,
     NodesReloadSecureSettingsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index fc72668d36413..22a667e7e8f6f 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
 import org.opensearch.action.support.nodes.BaseNodesRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -48,8 +49,9 @@
 /**
  * A request to get node (cluster) level stats.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesStatsRequest extends BaseNodesRequest {
 
     private CommonStatsFlags indices = new CommonStatsFlags();
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java
index e382278f5ddb8..58149e9a34a34 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
 import org.opensearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport builder for obtaining OpenSearch Node Stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesStatsRequestBuilder extends NodesOperationRequestBuilder<
     NodesStatsRequest,
     NodesStatsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java
index 8fbc52ad8a3f8..73a938568acc3 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
 import org.opensearch.cluster.ClusterName;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -47,8 +48,9 @@
 /**
  * Transport response for obtaining OpenSearch Node Stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesStatsResponse extends BaseNodesResponse implements ToXContentFragment {
 
     public NodesStatsResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java
index 360765e8f4803..183fb2a236148 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.node.tasks.cancel;
 
 import org.opensearch.action.support.tasks.BaseTasksRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.tasks.CancellableTask;
@@ -44,8 +45,9 @@
 /**
  * A request to cancel tasks
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CancelTasksRequest extends BaseTasksRequest {
 
     public static final String DEFAULT_REASON = "by user request";
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java
index ee19e8b104603..ac02dfdf7381f 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.tasks.TasksRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for the request to cancel tasks running on the specified nodes
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CancelTasksRequestBuilder extends TasksRequestBuilder {
 
     public CancelTasksRequestBuilder(OpenSearchClient client, CancelTasksAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
index 979489999cc6e..1f86c7c22c2eb 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.TaskOperationFailure;
 import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
@@ -49,8 +50,9 @@
 /**
  * Returns the list of tasks that were cancelled
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CancelTasksResponse extends ListTasksResponse {
 
     private static final ConstructingObjectParser PARSER = setupParser(
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
index 82902c55b1b0a..13c6d645d4c3a 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
 /**
  * A request to get node tasks
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetTaskRequest extends ActionRequest {
     private TaskId taskId = TaskId.EMPTY_TASK_ID;
     private boolean waitForCompletion = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java
index dd429da82ebf8..ea42e1770e7f1 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java
@@ -34,14 +34,16 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.tasks.TaskId;
 
 /**
  * Builder for the request to retrieve the list of tasks running on the specified nodes
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetTaskRequestBuilder extends ActionRequestBuilder {
     public GetTaskRequestBuilder(OpenSearchClient client, GetTaskAction action) {
         super(client, action, new GetTaskRequest());
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
index 9d4e6da8c7e62..80901373e14d5 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.node.tasks.get;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -48,8 +49,9 @@
 /**
  * Returns the list of tasks currently running on the nodes
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetTaskResponse extends ActionResponse implements ToXContentObject {
 
     private final TaskResult task;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
index 070b93c788ef0..6ee56b0da7884 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.node.tasks.list;
 
 import org.opensearch.action.support.tasks.BaseTasksRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -41,8 +42,9 @@
 /**
  * A request to get node tasks
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ListTasksRequest extends BaseTasksRequest {
 
     private boolean detailed = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java
index 45beb0dd899b5..a195b98d06e76 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.tasks.TasksRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for the request to retrieve the list of tasks running on the specified nodes
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ListTasksRequestBuilder extends TasksRequestBuilder {
 
     public ListTasksRequestBuilder(OpenSearchClient client, ListTasksAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index 693566391fb25..337151a2a9268 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -39,6 +39,7 @@
 import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.TriFunction;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -64,8 +65,9 @@
 /**
  * Returns the list of tasks currently running on the nodes
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ListTasksResponse extends BaseTasksResponse implements ToXContentObject {
     private static final String TASKS = "tasks";
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/TaskGroup.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/TaskGroup.java
index 0ca114ae0ed5c..fb23a41b8dc19 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/TaskGroup.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/tasks/list/TaskGroup.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.node.tasks.list;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.tasks.TaskInfo;
@@ -45,8 +46,9 @@
 /**
  * Information about a currently running task and all its subtasks.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TaskGroup implements ToXContentObject {
 
     private final TaskInfo task;
@@ -65,8 +67,9 @@ public static Builder builder(TaskInfo taskInfo) {
     /**
      * Builder for the Task Group
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
         private TaskInfo taskInfo;
         private List childTasks;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java
index e31b88ace953f..385f48d5690c1 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.node.usage;
 
 import org.opensearch.action.support.nodes.BaseNodesRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -41,8 +42,9 @@
 /**
  * Transport request for collecting OpenSearch telemetry
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesUsageRequest extends BaseNodesRequest {
 
     private boolean restActions;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java
index 7d1823b59dc04..ec1176ac634fa 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.ActionType;
 import org.opensearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport builder for collecting OpenSearch telemetry
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesUsageRequestBuilder extends NodesOperationRequestBuilder<
     NodesUsageRequest,
     NodesUsageResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java
index 575d88c10317c..7dc8a318b2cf4 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
 import org.opensearch.cluster.ClusterName;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -48,8 +49,9 @@
  * The response for the nodes usage api which contains the individual usage
  * statistics for all nodes queried.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodesUsageResponse extends BaseNodesResponse implements ToXContentFragment {
 
     public NodesUsageResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java
index 3b090415c175b..afa2058d1deba 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java
@@ -11,6 +11,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -30,8 +31,9 @@
 /**
  * Restore remote store request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.2.0")
 public class RestoreRemoteStoreRequest extends ClusterManagerNodeRequest implements ToXContentObject {
 
     private String[] indices = Strings.EMPTY_ARRAY;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java
index 66908889b0641..3de926c256f46 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreResponse.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.remotestore.restore;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -28,8 +29,9 @@
 /**
  * Contains information about remote store restores
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.2.0")
 public final class RestoreRemoteStoreResponse extends ActionResponse implements ToXContentObject {
 
     @Nullable
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java
index f292fcec7ccac..7bddd0deb373b 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.remotestore.stats;
 
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -22,8 +23,9 @@
 /**
  * Encapsulates all remote store stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.8.0")
 public class RemoteStoreStats implements Writeable, ToXContentFragment {
     /**
      * Stats related to Remote Segment Store operations
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java
index f09cf79c5154c..12c316adc75cc 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequest.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.remotestore.stats;
 
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -17,8 +18,9 @@
 /**
  * Encapsulates all remote store stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.8.0")
 public class RemoteStoreStatsRequest extends BroadcastRequest {
 
     private String[] shards;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestBuilder.java
index c31e4a1fd6178..4da700d3dc51b 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsRequestBuilder.java
@@ -10,13 +10,15 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
  * Builder for RemoteStoreStatsRequest
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.8.0")
 public class RemoteStoreStatsRequestBuilder extends BroadcastOperationRequestBuilder<
     RemoteStoreStatsRequest,
     RemoteStoreStatsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java
index 63a4c97d695b7..cad57d148770b 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.remotestore.stats;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -25,8 +26,9 @@
 /**
  * Remote Store stats response
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.8.0")
 public class RemoteStoreStatsResponse extends BroadcastResponse {
 
     private final RemoteStoreStats[] remoteStoreStats;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java
index 7e3f4cd95fc72..3e408c6114690 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -43,8 +44,9 @@
 /**
  * Transport request for cleaning up snapshot repositories
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CleanupRepositoryRequest extends AcknowledgedRequest {
 
     private String repository;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java
index 95c4fb372572f..34e42b157e627 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java
@@ -34,12 +34,14 @@
 import org.opensearch.action.ActionType;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport builder for cleaning up snapshot repositories
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CleanupRepositoryRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     CleanupRepositoryRequest,
     CleanupRepositoryResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java
index 99af862c769e8..e6790e8cbe708 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.action.admin.cluster.repositories.cleanup;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -46,8 +47,9 @@
 /**
  * Transport response for cleaning up snapshot repositories
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class CleanupRepositoryResponse extends ActionResponse implements ToXContentObject {
 
     private static final ObjectParser PARSER = new ObjectParser<>(
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java
index 352a3772e039b..04fdf22bee593 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -46,8 +47,9 @@
  * 
                              
  * The unregister repository command just unregisters the repository. No data is getting deleted from the repository.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteRepositoryRequest extends AcknowledgedRequest {
 
     private String name;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java
index ffef8d5b41979..6f5d0495e1c9f 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for unregister repository request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteRepositoryRequestBuilder extends AcknowledgedRequestBuilder<
     DeleteRepositoryRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java
index 6fb086f65497e..80a86f1b79209 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -45,8 +46,9 @@
 /**
  * Get repository request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetRepositoriesRequest extends ClusterManagerNodeReadRequest {
 
     private String[] repositories = Strings.EMPTY_ARRAY;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java
index 4b93aff4c25bc..b0c18f952b3df 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.ArrayUtils;
 
 /**
  * Get repository request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetRepositoriesRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     GetRepositoriesRequest,
     GetRepositoriesResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
index f8c8df25be532..f467b240aac31 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -50,8 +51,9 @@
 /**
  * Get repositories response
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetRepositoriesResponse extends ActionResponse implements ToXContentObject {
 
     private RepositoriesMetadata repositories;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
index 582f73f335b49..3fbd5743b5c8c 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.admin.cluster.crypto.CryptoSettings;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -57,8 +58,9 @@
  * Registers a repository with given name, type and settings. If the repository with the same name already
  * exists in the cluster, the new repository will replace the existing repository.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutRepositoryRequest extends AcknowledgedRequest implements ToXContentObject {
 
     private String name;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
index cf649ee6b4cbf..22aa6d7dc7c00 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
 
@@ -44,8 +45,9 @@
 /**
  * Register repository request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutRepositoryRequestBuilder extends AcknowledgedRequestBuilder<
     PutRepositoryRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java
index b84161e716f5d..ae6c92d8625ca 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -44,8 +45,9 @@
 /**
  * Verify repository request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class VerifyRepositoryRequest extends AcknowledgedRequest {
 
     private String name;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java
index c405fb9bc12cd..023f223700775 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for verify repository request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class VerifyRepositoryRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     VerifyRepositoryRequest,
     VerifyRepositoryResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
index d7af3478bdac3..12d6a4cca5683 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.repositories.verify;
 
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
@@ -54,8 +55,9 @@
 /**
  * Verify repository response
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class VerifyRepositoryResponse extends ActionResponse implements ToXContentObject {
 
     static final String NODES = "nodes";
@@ -64,8 +66,9 @@ public class VerifyRepositoryResponse extends ActionResponse implements ToXConte
     /**
      * Inner Node View
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class NodeView implements Writeable, ToXContentObject {
         private static final ObjectParser.NamedObjectParser PARSER;
         static {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
index a6addce14787d..1cefaa4866110 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.master.AcknowledgedRequest;
 import org.opensearch.cluster.routing.allocation.command.AllocationCommand;
 import org.opensearch.cluster.routing.allocation.command.AllocationCommands;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -45,8 +46,9 @@
 /**
  * Request to submit cluster reroute allocation commands
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterRerouteRequest extends AcknowledgedRequest {
     private AllocationCommands commands = new AllocationCommands();
     private boolean dryRun;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
index 01d52cb43320d..fc8310bdf7852 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.routing.allocation.command.AllocationCommand;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for a cluster reroute request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterRerouteRequestBuilder extends AcknowledgedRequestBuilder<
     ClusterRerouteRequest,
     ClusterRerouteResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
index a62029218ca25..ff01888040e82 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.routing.allocation.RoutingExplanations;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContent;
@@ -46,8 +47,9 @@
 /**
  * Response returned after a cluster reroute request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterRerouteResponse extends AcknowledgedResponse implements ToXContentObject {
 
     private final ClusterState state;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
index 6b62a5edd8c28..77aee99c2e902 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -55,8 +56,9 @@
 /**
  * Request for an update cluster settings action
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterUpdateSettingsRequest extends AcknowledgedRequest implements ToXContentObject {
 
     private static final ParseField PERSISTENT = new ParseField("persistent");
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
index 4d08c94f78b6a..53f1f17bbeb50 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
 
@@ -42,8 +43,9 @@
 /**
  * Builder for a cluster update settings request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterUpdateSettingsRequestBuilder extends AcknowledgedRequestBuilder<
     ClusterUpdateSettingsRequest,
     ClusterUpdateSettingsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
index 1c543260f7aee..2dfdb49736773 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.settings;
 
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -49,8 +50,9 @@
 /**
  * A response for a cluster update settings action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterUpdateSettingsResponse extends AcknowledgedResponse {
 
     private static final ParseField PERSISTENT = new ParseField("persistent");
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
index 84e9554932864..eaafca21e1894 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.shards;
 
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -45,8 +46,9 @@
 /**
  * Transport action for searching shard groups
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterSearchShardsGroup implements Writeable, ToXContentObject {
 
     private final ShardId shardId;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
index 7ddb945ad911e..62e05ebb37e28 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -47,8 +48,9 @@
 /**
  * Transport request for searching shards
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterSearchShardsRequest extends ClusterManagerNodeReadRequest
     implements
         IndicesRequest.Replaceable {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
index 674a2c2c36221..c4f8a29bbcf3d 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for searching shards
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterSearchShardsRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     ClusterSearchShardsRequest,
     ClusterSearchShardsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
index 2b24d870219bb..8ab6455bca4a7 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.shards;
 
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -47,8 +48,9 @@
 /**
  * Transport response for searching shards
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterSearchShardsResponse extends ActionResponse implements ToXContentObject {
 
     private final ClusterSearchShardsGroup[] groups;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java
index 7f69fe9fe3c72..72de28ca7e699 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequest.java
@@ -15,6 +15,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -32,8 +33,9 @@
 /**
  * Request to delete weights for weighted round-robin shard routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterDeleteWeightedRoutingRequest extends ClusterManagerNodeRequest {
     private static final Logger logger = LogManager.getLogger(ClusterDeleteWeightedRoutingRequest.class);
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequestBuilder.java
index bb34fea589534..e0d4e0ad2abed 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingRequestBuilder.java
@@ -10,12 +10,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Request builder to delete weights for weighted round-robin shard routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterDeleteWeightedRoutingRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     ClusterDeleteWeightedRoutingRequest,
     ClusterDeleteWeightedRoutingResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java
index 2a417e9f4287f..f6a18ae5055ae 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.shards.routing.weighted.delete;
 
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -17,8 +18,9 @@
 /**
  * Response from deleting weights for weighted round-robin search routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterDeleteWeightedRoutingResponse extends AcknowledgedResponse {
 
     ClusterDeleteWeightedRoutingResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java
index 7dcec15c750fc..937829de1db00 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -20,8 +21,9 @@
 /**
  * Request to get weights for weighted round-robin search routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterGetWeightedRoutingRequest extends ClusterManagerNodeReadRequest {
 
     String awarenessAttribute;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequestBuilder.java
index 82f4c1106461d..3cb5e7d3d07b9 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingRequestBuilder.java
@@ -10,12 +10,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Request builder to get weights for weighted round-robin search routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterGetWeightedRoutingRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     ClusterGetWeightedRoutingRequest,
     ClusterGetWeightedRoutingResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java
index 5ef14d0e00c49..b109ecb7de5d9 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/get/ClusterGetWeightedRoutingResponse.java
@@ -11,6 +11,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
 import org.opensearch.cluster.routing.WeightedRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -28,8 +29,9 @@
 /**
  * Response from fetching weights for weighted round-robin search routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterGetWeightedRoutingResponse extends ActionResponse implements ToXContentObject {
 
     private static final String WEIGHTS = "weights";
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java
index 8e8432a384aa5..c310e28610184 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequest.java
@@ -16,6 +16,7 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
 import org.opensearch.cluster.routing.WeightedRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -37,8 +38,9 @@
 /**
  * Request to update weights for weighted round-robin shard routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterPutWeightedRoutingRequest extends ClusterManagerNodeRequest {
     private static final Logger logger = LogManager.getLogger(ClusterPutWeightedRoutingRequest.class);
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestBuilder.java
index adfb2cf02f6d9..c520b0efd9838 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingRequestBuilder.java
@@ -11,12 +11,14 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.routing.WeightedRouting;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Request builder to update weights for weighted round-robin shard routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterPutWeightedRoutingRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     ClusterPutWeightedRoutingRequest,
     ClusterPutWeightedRoutingResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java
index cbf10aa74f8a2..4fee2f05a8715 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.cluster.shards.routing.weighted.put;
 
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -16,8 +17,9 @@
 /**
  * Response from updating weights for weighted round-robin search routing policy.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class ClusterPutWeightedRoutingResponse extends AcknowledgedResponse {
     public ClusterPutWeightedRoutingResponse(boolean acknowledged) {
         super(acknowledged);
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java
index 694b13e37bb03..4a05911610137 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -50,8 +51,9 @@
 /**
  * Transport request for cloning a snapshot
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CloneSnapshotRequest extends ClusterManagerNodeRequest
     implements
         IndicesRequest.Replaceable,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java
index 0f47d77d6a9d3..839a1b935ad1f 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java
@@ -37,13 +37,15 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 
 /**
  * Transport request builder for cloning a snapshot
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CloneSnapshotRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     CloneSnapshotRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 0cfc24994768a..0ad3071a99045 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -38,6 +38,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.Strings;
@@ -76,8 +77,9 @@
  * 
                            • must not contain invalid file name characters {@link Strings#INVALID_FILENAME_CHARS} 
                              • 
  * 
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CreateSnapshotRequest extends ClusterManagerNodeRequest
     implements
         IndicesRequest.Replaceable,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
index 40d440419819c..c378c416cc973 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
 
@@ -43,8 +44,9 @@
 /**
  * Create snapshot request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CreateSnapshotRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     CreateSnapshotRequest,
     CreateSnapshotResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index 1517112b9b3e1..5d25cf6cacfab 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.snapshots.create;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -51,8 +52,9 @@
 /**
  * Create snapshot response
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CreateSnapshotResponse extends ActionResponse implements ToXContentObject {
 
     private static final ObjectParser PARSER = new ObjectParser<>(
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
index d08c3033e7e10..21280381610f7 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -47,8 +48,9 @@
  * Delete snapshot request removes snapshots from the repository and cleans up all files that are associated with the snapshots.
  * All files that are shared with at least one other existing snapshot are left intact.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteSnapshotRequest extends ClusterManagerNodeRequest {
 
     private String repository;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java
index 1fa855265ffd4..f4da1ec0f7785 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Delete snapshot request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteSnapshotRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     DeleteSnapshotRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
index 8e2e2d3390058..7e95885c60f93 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -45,8 +46,9 @@
 /**
  * Get snapshot request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetSnapshotsRequest extends ClusterManagerNodeRequest {
 
     public static final String ALL_SNAPSHOTS = "_all";
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
index 3434f1cb47a99..983325aa575d7 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.ArrayUtils;
 
 /**
  * Get snapshots request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetSnapshotsRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     GetSnapshotsRequest,
     GetSnapshotsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index 5d5de4c7fa85e..6b0e8ba8a372f 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.snapshots.get;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
@@ -53,8 +54,9 @@
 /**
  * Get snapshots response
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetSnapshotsResponse extends ActionResponse implements ToXContentObject {
 
     @SuppressWarnings("unchecked")
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index c688b13ed9fdd..492ef86bb7843 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
@@ -63,15 +64,19 @@
 /**
  * Restore snapshot request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RestoreSnapshotRequest extends ClusterManagerNodeRequest implements ToXContentObject {
 
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RestoreSnapshotRequest.class);
 
     /**
      * Enumeration of possible storage types
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum StorageType {
         LOCAL("local"),
         REMOTE_SNAPSHOT("remote_snapshot");
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
index d9cca536d1c41..39eaadf3c8de6 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
 
@@ -44,8 +45,9 @@
 /**
  * Restore snapshot request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RestoreSnapshotRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     RestoreSnapshotRequest,
     RestoreSnapshotResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
index f6e50c69da3dc..c94645a6deb8f 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.snapshots.restore;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -53,8 +54,9 @@
 /**
  * Contains information about restores snapshot
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RestoreSnapshotResponse extends ActionResponse implements ToXContentObject {
 
     @Nullable
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
index 6e250962d1210..cd7f4b392c61d 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.action.admin.cluster.snapshots.status;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Stage for snapshotting an Index Shard
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum SnapshotIndexShardStage {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java
index fc3ffd4977da5..f991e90cb0728 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java
@@ -35,6 +35,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.action.support.broadcast.BroadcastShardResponse;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -56,8 +57,9 @@
 /**
  * Status for snapshotting an Index Shard
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotIndexShardStatus extends BroadcastShardResponse implements ToXContentFragment {
 
     private SnapshotIndexShardStage stage = SnapshotIndexShardStage.INIT;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java
index 9c2db62c33bd0..402fee76bc663 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.snapshots.status;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.ObjectParser;
@@ -54,8 +55,9 @@
 /**
  * Represents snapshot status of all shards in the index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotIndexStatus implements Iterable, ToXContentFragment {
 
     private final String index;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java
index ad514a13312ba..bfd0c23c579bc 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.snapshots.status;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.ToXContent;
@@ -47,8 +48,9 @@
 /**
  * Status of a snapshot shards
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotShardsStats implements ToXContentObject {
 
     private int initializingShards;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java
index babdb7540a314..f287e94edd0dc 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.snapshots.status;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -50,8 +51,9 @@
 /**
  * Stats for snapshots
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotStats implements Writeable, ToXContentObject {
 
     private long startTime;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java
index 22fca3c54c604..061e73f1094b5 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -45,8 +46,9 @@
 /**
  * Get snapshot status request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotsStatusRequest extends ClusterManagerNodeRequest {
 
     private String repository = "_all";
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java
index 55f156d4a470e..9377eca60e353 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.ArrayUtils;
 
 /**
  * Snapshots status request builder
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotsStatusRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     SnapshotsStatusRequest,
     SnapshotsStatusResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
index 2f0e5e4b4686f..df436a587c2ad 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.snapshots.status;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -51,8 +52,9 @@
 /**
  * Snapshot status response
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotsStatusResponse extends ActionResponse implements ToXContentObject {
 
     private final List snapshots;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java
index 7c937a1700db0..90a52f7406d57 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -46,8 +47,9 @@
 /**
  * Transport request for obtaining cluster state
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStateRequest extends ClusterManagerNodeReadRequest implements IndicesRequest.Replaceable {
 
     public static final TimeValue DEFAULT_WAIT_FOR_NODE_TIMEOUT = TimeValue.timeValueMinutes(1);
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestBuilder.java
index b9bfeca9f7386..01a49c15fc1ec 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequestBuilder.java
@@ -35,13 +35,15 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
  * Transport request builder for obtaining cluster state
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStateRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     ClusterStateRequest,
     ClusterStateResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java
index b947ad34a8cc2..d09105b2bd0a0 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -45,8 +46,9 @@
 /**
  * The response for getting the cluster state.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStateResponse extends ActionResponse {
 
     private ClusterName clusterName;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java
index 3372448bdacae..b7054ae99361b 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/AnalysisStats.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -61,8 +62,9 @@
 /**
  * Statistics about analysis usage.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class AnalysisStats implements ToXContentFragment, Writeable {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java
index 63ac76ae65783..26e554f44fca1 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.stats;
 
 import org.opensearch.action.admin.indices.stats.CommonStats;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.cache.query.QueryCacheStats;
@@ -50,8 +51,9 @@
 /**
  * Cluster Stats per index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStatsIndices implements ToXContentFragment {
 
     private int indexCount;
@@ -180,8 +182,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Inner Shard Stats
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ShardStats implements ToXContentFragment {
 
         int indices;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java
index 5689596763cef..b44e9cfc5c74a 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java
@@ -38,6 +38,7 @@
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.metrics.OperationStats;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.settings.Settings;
@@ -71,8 +72,9 @@
 /**
  * Per Node Cluster Stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStatsNodes implements ToXContentFragment {
 
     private final Counts counts;
@@ -214,8 +216,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Inner Counts
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Counts implements ToXContentFragment {
         static final String COORDINATING_ONLY = "coordinating_only";
 
@@ -282,8 +285,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Inner Operating System Stats
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class OsStats implements ToXContentFragment {
         final int availableProcessors;
         final int allocatedProcessors;
@@ -395,8 +399,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Inner Process Stats
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ProcessStats implements ToXContentFragment {
 
         final int count;
@@ -498,8 +503,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Inner JVM Stats
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class JvmStats implements ToXContentFragment {
 
         private final Map versions;
@@ -626,8 +632,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Inner JVM Version
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class JvmVersion {
         String version;
         String vmName;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java
index dc472c10f550b..6a99451c596ed 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.stats;
 
 import org.opensearch.action.support.nodes.BaseNodesRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -41,8 +42,9 @@
 /**
  * A request to get cluster level stats.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStatsRequest extends BaseNodesRequest {
 
     public ClusterStatsRequest(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java
index aaf5e3aeffeb8..0dcb03dc26d0e 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for obtaining cluster stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStatsRequestBuilder extends NodesOperationRequestBuilder<
     ClusterStatsRequest,
     ClusterStatsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java
index 1b609b1f7556d..cc002b689a2a5 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -37,6 +37,7 @@
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -50,8 +51,9 @@
 /**
  * Transport response for obtaining cluster stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStatsResponse extends BaseNodesResponse implements ToXContentFragment {
 
     final ClusterStatsNodes nodesStats;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java
index f833c52493e00..f73c363b2ea60 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/IndexFeatureStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.stats;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * Statistics about an index feature.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class IndexFeatureStats implements ToXContent, Writeable {
 
     final String name;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java
index 64218765a57e6..8e6fdb02b1f22 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/MappingStats.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -58,8 +59,9 @@
 /**
  * Usage statistics about mappings usage.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class MappingStats implements ToXContentFragment, Writeable {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
index 7d92162015950..0bb4f3625ddad 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -44,8 +45,9 @@
 /**
  * Transport request for deleting stored scripts
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteStoredScriptRequest extends AcknowledgedRequest {
 
     private String id;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
index 34e0d429f2098..cbadde386d5a9 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for deleting stored scripts
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<
     DeleteStoredScriptRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
index 70384b5fb648e..25bc3ecd6b7ee 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -44,8 +45,9 @@
 /**
  * Transport request for getting stored script
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetStoredScriptRequest extends ClusterManagerNodeReadRequest {
 
     protected String id;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestBuilder.java
index ae969963be62f..ca0bd32f1f38b 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for getting stored script
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetStoredScriptRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     GetStoredScriptRequest,
     GetStoredScriptResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
index cf5a3ec44e560..a81faff2abb03 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.cluster.storedscripts;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -53,8 +54,9 @@
 /**
  * Transport response for getting stored script
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetStoredScriptResponse extends ActionResponse implements StatusToXContentObject {
 
     public static final ParseField _ID_PARSE_FIELD = new ParseField("_id");
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
index f45bee955da02..8731b18fff338 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.Strings;
@@ -54,8 +55,9 @@
 /**
  * Transport request for putting stored script
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutStoredScriptRequest extends AcknowledgedRequest implements ToXContentFragment {
 
     private String id;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
index 2a06cd23c10b6..46773177e9a74 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
@@ -35,14 +35,16 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaType;
 
 /**
  * Transport request builder for putting stored script
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<
     PutStoredScriptRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java
index 83e8b93b32e0f..16103f02be596 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -41,8 +42,9 @@
 /**
  * Transport request for getting pending cluster tasks
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PendingClusterTasksRequest extends ClusterManagerNodeReadRequest {
 
     public PendingClusterTasksRequest() {}
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java
index b5e77f291a701..c932c2e91f314 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for getting pending cluster tasks
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PendingClusterTasksRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     PendingClusterTasksRequest,
     PendingClusterTasksResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
index 118d2cf3065e6..9f4568c88b273 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.tasks;
 
 import org.opensearch.cluster.service.PendingClusterTask;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
 /**
  * Transport response for getting pending cluster tasks
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PendingClusterTasksResponse extends ActionResponse implements Iterable, ToXContentObject {
 
     private final List pendingTasks;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java
index 94dbb5ff46a02..0b56216790d94 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/Alias.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.OpenSearchGenerationException;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -54,8 +55,9 @@
 /**
  * Represents an alias, to be associated with an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Alias implements Writeable, ToXContentFragment {
 
     private static final ParseField FILTER = new ParseField("filter");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 2899e791604a5..bc27c70282368 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -38,6 +38,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
 import org.opensearch.cluster.metadata.AliasAction;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.ParsingException;
@@ -72,8 +73,9 @@
 /**
  * A request to add/remove aliases for one or more indices.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesAliasesRequest extends AcknowledgedRequest implements ToXContentObject {
 
     private List allAliasActions = new ArrayList<>();
@@ -95,8 +97,9 @@ public IndicesAliasesRequest() {}
     /**
      * Request to take one or more actions on one or more indexes and alias combinations.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class AliasActions implements AliasesRequest, Writeable, ToXContentObject {
 
         private static final ParseField INDEX = new ParseField("index");
@@ -118,8 +121,9 @@ public static class AliasActions implements AliasesRequest, Writeable, ToXConten
         /**
          * The type of request.
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public enum Type {
             ADD((byte) 0, AliasActions.ADD),
             REMOVE((byte) 1, AliasActions.REMOVE),
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java
index 13c57cc781925..d262c9cd42ce9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.index.query.QueryBuilder;
 
 import java.util.Map;
@@ -42,8 +43,9 @@
 /**
  * Builder for request to modify many aliases at once.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesAliasesRequestBuilder extends AcknowledgedRequestBuilder<
     IndicesAliasesRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java
index 28894baa629b3..00d754c8fb029 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequest.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.AliasesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
 /**
  * Transport request for listing index aliases
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetAliasesRequest extends ClusterManagerNodeReadRequest implements AliasesRequest {
 
     private String[] indices = Strings.EMPTY_ARRAY;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java
index aecbd689a647c..e9a15e9f9dfb3 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java
@@ -33,12 +33,14 @@
 package org.opensearch.action.admin.indices.alias.get;
 
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for listing index aliases
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetAliasesRequestBuilder extends BaseAliasesRequestBuilder {
 
     public GetAliasesRequestBuilder(OpenSearchClient client, GetAliasesAction action, String... aliases) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java
index e50540e6b9aea..71cbbe2c6594f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/GetAliasesResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.alias.get;
 
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
 /**
  * Transport response for listing index aliases
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetAliasesResponse extends ActionResponse {
 
     private final Map> aliases;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java
index 274bb8bd15e08..a70f12bed4f1f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeAction.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.ActionType;
 import org.opensearch.action.support.single.shard.SingleShardRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
@@ -62,8 +63,9 @@
 /**
  * Transport action for analyzing text
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AnalyzeAction extends ActionType {
 
     public static final AnalyzeAction INSTANCE = new AnalyzeAction();
@@ -77,8 +79,9 @@ private AnalyzeAction() {
      * A request to analyze a text associated with a specific index. Allow to provide
      * the actual analyzer name to perform the analysis with.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Request extends SingleShardRequest {
 
         private String[] text;
@@ -308,8 +311,9 @@ public static Request fromXContent(XContentParser parser, String index) throws I
     /**
      * Inner Response
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Response extends ActionResponse implements ToXContentObject {
 
         private final DetailAnalyzeResponse detail;
@@ -404,8 +408,9 @@ static final class Fields {
     /**
      * Inner Analyze Token
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class AnalyzeToken implements Writeable, ToXContentObject {
         private final String term;
         private final int startOffset;
@@ -542,8 +547,9 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Inner Detail Analyze Response
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class DetailAnalyzeResponse implements Writeable, ToXContentFragment {
 
         private final boolean customAnalyzer;
@@ -709,8 +715,9 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Inner Analyze Token List
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class AnalyzeTokenList implements Writeable, ToXContentObject {
         private final String name;
         private final AnalyzeToken[] tokens;
@@ -783,8 +790,9 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Inner character filtered text
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class CharFilteredText implements Writeable, ToXContentObject {
         private final String name;
         private final String[] texts;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
index a7f21b2af16fc..b0240a4db82cd 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
@@ -33,14 +33,16 @@
 
 import org.opensearch.action.support.single.shard.SingleShardOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.Map;
 
 /**
  * Transport request builder for analyzing text
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AnalyzeRequestBuilder extends SingleShardOperationRequestBuilder<
     AnalyzeAction.Request,
     AnalyzeAction.Response,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java
index 3b69a0980e97f..57266b5aec58f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -43,8 +44,9 @@
 /**
  * Transport request for clearing cache
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClearIndicesCacheRequest extends BroadcastRequest {
 
     private boolean queryCache = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java
index e0513e77a5aa5..074e2ce0b35eb 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for clearing cache
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClearIndicesCacheRequestBuilder extends BroadcastOperationRequestBuilder<
     ClearIndicesCacheRequest,
     ClearIndicesCacheResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java
index 0c6c5ca27b24d..6fe180d900311 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.cache.clear;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
@@ -45,8 +46,9 @@
 /**
  * The response of a clear cache action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClearIndicesCacheResponse extends BroadcastResponse {
 
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java
index 0f98550343a13..e785c31c4a0b9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.util.CollectionUtils;
@@ -48,8 +49,9 @@
 /**
  * A request to close an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CloseIndexRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable {
 
     private String[] indices;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java
index b3b53a0043c70..92c32c9ace490 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java
@@ -36,12 +36,14 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for close index request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CloseIndexRequestBuilder extends AcknowledgedRequestBuilder {
 
     public CloseIndexRequestBuilder(OpenSearchClient client, CloseIndexAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java
index df6d0e4c9be39..2e0c5cb5842b4 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java
@@ -34,6 +34,7 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.support.master.ShardsAcknowledgedResponse;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -54,8 +55,9 @@
 /**
  * Transport response for closing an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CloseIndexResponse extends ShardsAcknowledgedResponse {
 
     private final List indices;
@@ -99,8 +101,9 @@ public String toString() {
     /**
      * Inner index result
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class IndexResult implements Writeable, ToXContentFragment {
 
         private final Index index;
@@ -199,8 +202,9 @@ public String toString() {
     /**
      * Shard Result from Close Index Response
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ShardResult implements Writeable, ToXContentFragment {
 
         private final int id;
@@ -257,8 +261,9 @@ public String toString() {
         /**
          * Inner Failure if something goes wrong
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public static class Failure extends DefaultShardOperationFailedException {
 
             private @Nullable String nodeId;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
index d048a3008f588..e5dbefc3dba97 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java
@@ -42,6 +42,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.XContentFactory;
@@ -80,8 +81,9 @@
  * @see org.opensearch.client.Requests#createIndexRequest(String)
  * @see CreateIndexResponse
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CreateIndexRequest extends AcknowledgedRequest implements IndicesRequest {
 
     public static final ParseField MAPPINGS = new ParseField("mappings");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
index 3493ee06827c1..b233f45422967 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.XContentType;
@@ -48,8 +49,9 @@
 /**
  * Builder for a create index request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<
     CreateIndexRequest,
     CreateIndexResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java
index 1b3ad48402eed..3258ffd8672a1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.create;
 
 import org.opensearch.action.support.master.ShardsAcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -49,8 +50,9 @@
 /**
  * A response for a create index action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CreateIndexResponse extends ShardsAcknowledgedResponse {
 
     private static final ParseField INDEX = new ParseField("index");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java
index 0443325f82778..c5c03c93785d2 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -44,8 +45,9 @@
  * Represents a request to delete a particular dangling index, specified by its UUID. The {@link #acceptDataLoss}
  * flag must also be explicitly set to true, or later validation will fail.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteDanglingIndexRequest extends AcknowledgedRequest {
     private final String indexUUID;
     private final boolean acceptDataLoss;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java
index 590f08a82c1d2..2702b6a05c4bb 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -46,8 +47,9 @@
  * by its UUID. The {@link #acceptDataLoss} flag must also be
  * explicitly set to true, or later validation will fail.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ImportDanglingIndexRequest extends AcknowledgedRequest {
     private final String indexUUID;
     private final boolean acceptDataLoss;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java
index adbfe9b760ed8..119c4acbf4160 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.dangling.list;
 
 import org.opensearch.action.support.nodes.BaseNodesRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -42,8 +43,9 @@
 /**
  * Transport request for listing a dangling indices
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ListDanglingIndicesRequest extends BaseNodesRequest {
     /**
      * Filter the response by index UUID. Leave as null to find all indices.
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
index 384ae2e028bba..be63bee6312fe 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.admin.indices.dangling.DanglingIndexInfo;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
 import org.opensearch.cluster.ClusterName;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -58,8 +59,9 @@
  * information for each dangling index is presented under the "dangling_indices" key. If any nodes
  * in the cluster failed to answer, the details are presented under the "_nodes.failures" key.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ListDanglingIndicesResponse extends BaseNodesResponse implements StatusToXContentObject {
 
     public ListDanglingIndicesResponse(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java
index 14b29c4b2bbba..4c1690d25fbd9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java
@@ -47,6 +47,7 @@
 import org.opensearch.cluster.metadata.MetadataCreateDataStreamService;
 import org.opensearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.Strings;
@@ -61,8 +62,9 @@
 /**
  * Transport action for creating a datastream
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CreateDataStreamAction extends ActionType {
 
     public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction();
@@ -75,8 +77,9 @@ private CreateDataStreamAction() {
     /**
      * Request for Creating Data Stream
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Request extends AcknowledgedRequest implements IndicesRequest {
 
         private final String name;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java
index 190194e071bc4..6b0aec6a31839 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java
@@ -53,6 +53,7 @@
 import org.opensearch.cluster.service.ClusterManagerTaskThrottler;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Priority;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.common.unit.TimeValue;
@@ -78,8 +79,9 @@
 /**
  * Transport action for deleting a datastream
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteDataStreamAction extends ActionType {
 
     private static final Logger logger = LogManager.getLogger(DeleteDataStreamAction.class);
@@ -94,8 +96,9 @@ private DeleteDataStreamAction() {
     /**
      * Request for deleting data streams
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Request extends ClusterManagerNodeRequest implements IndicesRequest.Replaceable {
 
         private String[] names;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java
index 6a7967d31653c..1db4e85887c23 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java
@@ -51,6 +51,7 @@
 import org.opensearch.cluster.metadata.MetadataIndexTemplateService;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.ParseField;
@@ -76,8 +77,9 @@
 /**
  * Transport action for getting a datastream
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetDataStreamAction extends ActionType {
 
     public static final GetDataStreamAction INSTANCE = new GetDataStreamAction();
@@ -90,8 +92,9 @@ private GetDataStreamAction() {
     /**
      * Request for getting data streams
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Request extends ClusterManagerNodeReadRequest implements IndicesRequest.Replaceable {
 
         private String[] names;
@@ -156,16 +159,18 @@ public IndicesRequest indices(String... indices) {
     /**
      * Response for getting data streams
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Response extends ActionResponse implements ToXContentObject {
         public static final ParseField DATASTREAMS_FIELD = new ParseField("data_streams");
 
         /**
          * Data streams information
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public static class DataStreamInfo extends AbstractDiffable implements ToXContentObject {
 
             public static final ParseField STATUS_FIELD = new ParseField("status");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java
index 006da6b3cbb09..5fbefbc6e1591 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.util.CollectionUtils;
@@ -47,8 +48,9 @@
 /**
  * A request to delete an index. Best created with {@link org.opensearch.client.Requests#deleteIndexRequest(String)}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteIndexRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable {
 
     private String[] indices;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java
index 33f6342e94139..6cf0920f8570f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java
@@ -36,12 +36,14 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for deleting an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteIndexRequestBuilder extends AcknowledgedRequestBuilder<
     DeleteIndexRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java
index f9bd849e549b7..91fde3ec62d7b 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -47,8 +48,9 @@
 /**
  * Transport request for checking if an index exists
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesExistsRequest extends ClusterManagerNodeReadRequest implements IndicesRequest.Replaceable {
 
     private String[] indices = Strings.EMPTY_ARRAY;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java
index 8459bbd8b874e..2e0f28cb7e3f4 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for checking if an index exists
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesExistsRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     IndicesExistsRequest,
     IndicesExistsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java
index 4ac85492229f0..a457cca74f897 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/IndicesExistsResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.exists.indices;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -41,8 +42,9 @@
 /**
  * Transport response for checking if an index exists
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesExistsResponse extends ActionResponse {
 
     private boolean exists;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java
index c8b28efc5f294..f8cf6ab72e038 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -52,8 +53,9 @@
  * @see org.opensearch.client.IndicesAdminClient#flush(FlushRequest)
  * @see FlushResponse
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FlushRequest extends BroadcastRequest {
 
     private boolean force = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequestBuilder.java
index d0cbd1d27fba6..50d7a78c919f1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for flushing one or more indices
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FlushRequestBuilder extends BroadcastOperationRequestBuilder {
 
     public FlushRequestBuilder(OpenSearchClient client, FlushAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java
index 4135654e66271..3881a839a6dcd 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/flush/FlushResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.flush;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
@@ -45,8 +46,9 @@
 /**
  * A response to flush action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FlushResponse extends BroadcastResponse {
 
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("flush", true, arg -> {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java
index 89e5a57094a96..f38b49f434261 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequest.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.index.engine.Engine;
@@ -54,8 +55,9 @@
  * @see org.opensearch.client.IndicesAdminClient#forceMerge(ForceMergeRequest)
  * @see ForceMergeResponse
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ForceMergeRequest extends BroadcastRequest {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java
index cff05f194cac4..d8a618a1828ad 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A request to force merge one or more indices. In order to force merge all
@@ -42,8 +43,9 @@
  * merge down to. By default, will cause the force merge process to merge down
  * to half the configured number of segments.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ForceMergeRequestBuilder extends BroadcastOperationRequestBuilder<
     ForceMergeRequest,
     ForceMergeResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java
index c5316f3375ad5..e6a7fe0025b87 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/forcemerge/ForceMergeResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.forcemerge;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
@@ -47,8 +48,9 @@
 /**
  * A response for force merge action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ForceMergeResponse extends BroadcastResponse {
 
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
index 0b2084865e23e..47c59791edf04 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.info.ClusterInfoRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.ArrayUtils;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -43,14 +44,16 @@
 /**
  * A request to retrieve information about an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetIndexRequest extends ClusterInfoRequest {
     /**
      * The features to get.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Feature {
         ALIASES((byte) 0),
         MAPPINGS((byte) 1),
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequestBuilder.java
index 3019191e5570e..e97319abe5f98 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.master.info.ClusterInfoRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder to get information about an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetIndexRequestBuilder extends ClusterInfoRequestBuilder {
 
     public GetIndexRequestBuilder(OpenSearchClient client, GetIndexAction action, String... indices) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
index c135b511c3315..5a237b8d3470f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.AliasMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
@@ -57,8 +58,9 @@
 /**
  * A response for a get index action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetIndexResponse extends ActionResponse implements ToXContentObject {
 
     private Map mappings = Map.of();
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
index 6a1d04efa9714..f18f973c07959 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -50,8 +51,9 @@
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should go to that client class as well.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetFieldMappingsRequest extends ActionRequest implements IndicesRequest.Replaceable {
 
     protected boolean local = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java
index ebc0c015c5140..d379bfbecafd0 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java
@@ -35,13 +35,15 @@
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.ArrayUtils;
 
 /**
  * A helper class to build {@link GetFieldMappingsRequest} objects
  *
- * @opensearch.internal
- **/
+ * @opensearch.api
+ */
+@PublicApi(since = "1.0.0")
 public class GetFieldMappingsRequestBuilder extends ActionRequestBuilder {
 
     public GetFieldMappingsRequestBuilder(OpenSearchClient client, GetFieldMappingsAction action, String... indices) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index 94c88e30295a8..86533f14e83e1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.mapping.get;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -65,8 +66,9 @@
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should go to that client class as well.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetFieldMappingsResponse extends ActionResponse implements ToXContentObject {
 
     private static final ParseField MAPPINGS = new ParseField("mappings");
@@ -178,8 +180,9 @@ private void addFieldMappingsToBuilder(XContentBuilder builder, Params params, M
     /**
      * Metadata for field mappings for toXContent
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class FieldMappingMetadata implements ToXContentFragment {
 
         private static final ParseField FULL_NAME = new ParseField("full_name");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java
index 3988b0dd5a508..cd0ecdb30e5fa 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.info.ClusterInfoRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -41,8 +42,9 @@
 /**
  * Transport request to get field mappings.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetMappingsRequest extends ClusterInfoRequest {
 
     public GetMappingsRequest() {}
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java
index 85bf8c2ffd9c6..36ca1cb088cb5 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.master.info.ClusterInfoRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder to get field mappings.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetMappingsRequestBuilder extends ClusterInfoRequestBuilder<
     GetMappingsRequest,
     GetMappingsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java
index e59f08c4fa162..56c979c20b6d9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetMappingsResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.MappingMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
@@ -52,8 +53,9 @@
 /**
  * Transport response to get field mappings.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetMappingsResponse extends ActionResponse implements ToXContentFragment {
 
     private static final ParseField MAPPINGS = new ParseField("mappings");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 32f751ceb1c5a..8122db3278795 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -39,6 +39,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
 import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.common.Strings;
@@ -75,8 +76,9 @@
  * @see org.opensearch.client.IndicesAdminClient#putMapping(PutMappingRequest)
  * @see AcknowledgedResponse
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutMappingRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable, ToXContentObject {
 
     private static final Set RESERVED_FIELDS = Set.of(
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
index 9d703bf428c8e..d44b243bb0edb 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -45,8 +46,9 @@
 /**
  * Builder for a put mapping request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutMappingRequestBuilder extends AcknowledgedRequestBuilder<
     PutMappingRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java
index 16451e311e7d3..f48ec1ae6fb71 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.util.CollectionUtils;
@@ -49,8 +50,9 @@
 /**
  * A request to open an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenIndexRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable {
 
     private String[] indices;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java
index bf09c3f173491..19770255b0ee1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java
@@ -36,12 +36,14 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for for open index request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenIndexRequestBuilder extends AcknowledgedRequestBuilder {
 
     public OpenIndexRequestBuilder(OpenSearchClient client, OpenIndexAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java
index bd96a1071c129..78af1abc3ce31 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.open;
 
 import org.opensearch.action.support.master.ShardsAcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
@@ -43,8 +44,9 @@
 /**
  * A response for a open index action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenIndexResponse extends ShardsAcknowledgedResponse {
 
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java
index b5097f96fe52b..1fb8514cbf48c 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
 import org.opensearch.cluster.metadata.IndexMetadata.APIBlock;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.util.CollectionUtils;
@@ -49,8 +50,9 @@
 /**
  * A request to add a block to an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AddIndexBlockRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable {
 
     private final APIBlock block;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java
index 8322ba19f433e..ebcdf700d3b6e 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java
@@ -36,12 +36,14 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.metadata.IndexMetadata.APIBlock;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Builder for add index block request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AddIndexBlockRequestBuilder extends AcknowledgedRequestBuilder<
     AddIndexBlockRequest,
     AddIndexBlockResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java
index 13cee3f8e0159..3ab64fa55af8b 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java
@@ -34,6 +34,7 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.support.master.ShardsAcknowledgedResponse;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -54,8 +55,9 @@
 /**
  * Transport response to open an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AddIndexBlockResponse extends ShardsAcknowledgedResponse {
 
     private final List indices;
@@ -99,8 +101,9 @@ public String toString() {
     /**
      * Result for adding a block
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class AddBlockResult implements Writeable, ToXContentFragment {
 
         private final Index index;
@@ -199,8 +202,9 @@ public String toString() {
     /**
      * Per shard result for adding a block
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class AddBlockShardResult implements Writeable, ToXContentFragment {
 
         private final int id;
@@ -258,8 +262,9 @@ public String toString() {
         /**
          * Contains failure information
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public static class Failure extends DefaultShardOperationFailedException {
 
             private @Nullable String nodeId;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java
index 69b20b697dd9a..aca98a6d9c571 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -43,8 +44,9 @@
 /**
  * Request for recovery information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RecoveryRequest extends BroadcastRequest {
 
     private boolean detailed = false;       // Provides extra details in the response
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequestBuilder.java
index 99a1fb430fb28..2f44a5f2df04a 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Recovery information request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RecoveryRequestBuilder extends BroadcastOperationRequestBuilder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java
index e6440fd95aa39..27b6b334ef4b4 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/recovery/RecoveryResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.recovery;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -48,8 +49,9 @@
 /**
  * Information regarding the recovery state of indices and their associated shards.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RecoveryResponse extends BroadcastResponse {
 
     private final Map> shardRecoveryStates;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java
index c6e230cc66373..c8000cbc40da8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.refresh;
 
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -46,8 +47,9 @@
  * @see org.opensearch.client.IndicesAdminClient#refresh(RefreshRequest)
  * @see RefreshResponse
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RefreshRequest extends BroadcastRequest {
 
     public RefreshRequest(String... indices) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequestBuilder.java
index 5b27ae13f24be..ebafc726bfd39 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshRequestBuilder.java
@@ -34,14 +34,16 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A refresh request making all operations performed since the last refresh available for search. The (near) real-time
  * capabilities depends on the index engine used. For example, the internal one requires refresh to be called, but by
  * default a refresh is scheduled periodically.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RefreshRequestBuilder extends BroadcastOperationRequestBuilder {
 
     public RefreshRequestBuilder(OpenSearchClient client, RefreshAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java
index 4c89962dfec5e..30351b8983717 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/refresh/RefreshResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.refresh;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
@@ -45,8 +46,9 @@
 /**
  * The response of a refresh action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RefreshResponse extends BroadcastResponse {
 
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("refresh", true, arg -> {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java
index b82b68f6f9489..9a913c6bcafff 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -19,8 +20,9 @@
 /**
  * Request for Segment Replication stats information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SegmentReplicationStatsRequest extends BroadcastRequest {
     private boolean detailed = false;       // Provides extra details in the response
     private boolean activeOnly = false;     // Only reports on active segment replication events
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequestBuilder.java
index 7e68d2ac59f07..9f00bff414cf5 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsRequestBuilder.java
@@ -10,12 +10,14 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Segment Replication stats information request builder.
  *
- * @opensearch.internal
+  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SegmentReplicationStatsRequestBuilder extends BroadcastOperationRequestBuilder<
     SegmentReplicationStatsRequest,
     SegmentReplicationStatsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java
index f1b6b90e0cae1..e65e13a945abd 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationStatsResponse.java
@@ -9,6 +9,7 @@
 package org.opensearch.action.admin.indices.replication;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -24,8 +25,9 @@
 /**
  * Stats Information regarding the Segment Replication state of indices and their associated shards.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SegmentReplicationStatsResponse extends BroadcastResponse {
     private final Map> replicationStats;
 
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java
index 98c2446e17998..e20e4c2d868a8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java
@@ -49,6 +49,7 @@
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.util.concurrent.CountDown;
 import org.opensearch.core.ParseField;
@@ -82,8 +83,9 @@
 /**
  * Transport action to resolve an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ResolveIndexAction extends ActionType {
 
     public static final ResolveIndexAction INSTANCE = new ResolveIndexAction();
@@ -96,8 +98,9 @@ private ResolveIndexAction() {
     /**
      * Request for resolving an index
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Request extends ActionRequest implements IndicesRequest.Replaceable {
 
         public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpen();
@@ -196,8 +199,9 @@ public String getName() {
     /**
      * The resolved index
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ResolvedIndex extends ResolvedIndexAbstraction implements Writeable, ToXContentObject {
 
         static final ParseField ALIASES_FIELD = new ParseField("aliases");
@@ -284,8 +288,9 @@ public int hashCode() {
     /**
      * The resolved index alias
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ResolvedAlias extends ResolvedIndexAbstraction implements Writeable, ToXContentObject {
 
         static final ParseField INDICES_FIELD = new ParseField("indices");
@@ -346,8 +351,9 @@ public int hashCode() {
     /**
      * The resolved data stream
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ResolvedDataStream extends ResolvedIndexAbstraction implements Writeable, ToXContentObject {
 
         static final ParseField BACKING_INDICES_FIELD = new ParseField("backing_indices");
@@ -418,8 +424,9 @@ public int hashCode() {
     /**
      * Response for resolving an index
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Response extends ActionResponse implements ToXContentObject {
 
         static final ParseField INDICES_FIELD = new ParseField("indices");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java
index e014d6d703500..4d0b7fc8c13c7 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.rollover;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.NamedWriteable;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.xcontent.ToXContentFragment;
@@ -42,8 +43,9 @@
 /**
  * Base class for rollover request conditions
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class Condition implements NamedWriteable, ToXContentFragment {
 
     protected T value;
@@ -96,8 +98,9 @@ public String name() {
     /**
      * Holder for index stats used to evaluate conditions
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Stats {
         public final long numDocs;
         public final long indexCreated;
@@ -113,8 +116,9 @@ public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
     /**
      * Holder for evaluated condition result
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Result {
         public final Condition condition;
         public final boolean matched;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
index 353cdbbbc840c..68c0076bbd302 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -58,8 +59,9 @@
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should also go to that client class.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest {
 
     private static final ObjectParser PARSER = new ObjectParser<>("rollover");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index 9e6b8518e92d3..acac7102edbc7 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.unit.ByteSizeValue;
@@ -42,8 +43,9 @@
 /**
  * Transport request to rollover an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RolloverRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     RolloverRequest,
     RolloverResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java
index a66fcc9e9bcf2..b7df35cd480bb 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.rollover;
 
 import org.opensearch.action.support.master.ShardsAcknowledgedResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -55,8 +56,9 @@
  * Note: there is a new class with the same name for the Java HLRC that uses a typeless format.
  * Any changes done to this class should also go to that client class.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class RolloverResponse extends ShardsAcknowledgedResponse implements ToXContentObject {
 
     private static final ParseField NEW_INDEX = new ParseField("new_index");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexSegments.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexSegments.java
index 7249bc5e9d3ba..4b37da2c99850 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexSegments.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexSegments.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.action.admin.indices.segments;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -41,8 +43,9 @@
 /**
  * List of Index Segments
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexSegments implements Iterable {
 
     private final String index;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java
index a6caf0649fde1..8fdc050511050 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndexShardSegments.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.segments;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.index.shard.ShardId;
 
 import java.util.Arrays;
@@ -40,8 +41,9 @@
 /**
  * List of Index Shard Segments
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexShardSegments implements Iterable {
 
     private final ShardId shardId;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java
index 36f5979552011..648f58dada4f9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.search.SortedSetSortField;
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -55,8 +56,9 @@
 /**
  * Transport response for retrieving indices segment information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesSegmentResponse extends BroadcastResponse {
 
     private final ShardSegments[] shards;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java
index 03a41cd21572f..aff2b383df08f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.segments;
 
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -42,8 +43,9 @@
 /**
  * Transport request for retrieving indices segment information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesSegmentsRequest extends BroadcastRequest {
 
     protected boolean verbose = false;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java
index 4b758e1f4bfb1..579b6d997acd9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for retrieving indices segment information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesSegmentsRequestBuilder extends BroadcastOperationRequestBuilder<
     IndicesSegmentsRequest,
     IndicesSegmentResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java
index a4dc901544e0b..84edec384b68a 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/PitSegmentsRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -25,7 +26,10 @@
 
 /**
  * Transport request for retrieving PITs segment information
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class PitSegmentsRequest extends BroadcastRequest {
     private boolean verbose = false;
     private final List pitIds = new ArrayList<>();
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java
index 90317542244ff..09adda4d79108 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/ShardSegments.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.segments;
 
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -45,8 +46,9 @@
 /**
  * Collection of shard segments
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ShardSegments implements Writeable, Iterable {
 
     private final ShardRouting shardRouting;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java
index 01383b6b6545d..547cfa8c3bce3 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.ValidateActions;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -48,8 +49,9 @@
 /**
  * Transport request for getting index segments
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetSettingsRequest extends ClusterManagerNodeReadRequest implements IndicesRequest.Replaceable {
 
     private String[] indices = Strings.EMPTY_ARRAY;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java
index 84cd4e8682e93..5ba42c05dccf0 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java
@@ -35,13 +35,15 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.ArrayUtils;
 
 /**
  * Transport request builder for getting index segments
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetSettingsRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     GetSettingsRequest,
     GetSettingsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java
index 61c9b68629194..695c98684f0f4 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/GetSettingsResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.settings.get;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.core.action.ActionResponse;
@@ -53,8 +54,9 @@
 /**
  * Transport response for getting index segments
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetSettingsResponse extends ActionResponse implements ToXContentObject {
 
     private final Map indexToSettings;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
index 43571dc8220f9..45172e313dfcc 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -60,8 +61,9 @@
 /**
  * Request for an update index settings action
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpdateSettingsRequest extends AcknowledgedRequest
     implements
         IndicesRequest.Replaceable,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
index 7501f0c7798de..08d7a240aa007 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
 
@@ -44,8 +45,9 @@
 /**
  * Builder for an update index settings request
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpdateSettingsRequestBuilder extends AcknowledgedRequestBuilder<
     UpdateSettingsRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java
index bc9633f2bd2db..b986ef3c62e73 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java
@@ -37,12 +37,14 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Request builder for {@link IndicesShardStoresRequest}
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesShardStoreRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     IndicesShardStoresRequest,
     IndicesShardStoresResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java
index d3261bea68f38..ea07325c35c3b 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
 import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
 /**
  * Request for {@link IndicesShardStoresAction}
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesShardStoresRequest extends ClusterManagerNodeReadRequest
     implements
         IndicesRequest.Replaceable {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index 98132b25a6d99..c2f373fcbd6a1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.OpenSearchException;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -53,15 +54,17 @@
  * Consists of {@link StoreStatus}s for requested indices grouped by
  * indices and shard ids and a list of encountered node {@link Failure}s
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesShardStoresResponse extends ActionResponse implements ToXContentFragment {
 
     /**
      * Shard store information from a node
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class StoreStatus implements Writeable, ToXContentFragment, Comparable {
         private final DiscoveryNode node;
         private final String allocationId;
@@ -70,7 +73,10 @@ public static class StoreStatus implements Writeable, ToXContentFragment, Compar
 
         /**
          * The status of the shard store with respect to the cluster
+         *
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public enum AllocationStatus {
 
             /**
@@ -232,8 +238,9 @@ public int compareTo(StoreStatus other) {
     /**
      * Single node failure while retrieving shard store information
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Failure extends DefaultShardOperationFailedException {
         private String nodeId;
 
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java
index a24a33006ac1e..a5225f2243876 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java
@@ -40,6 +40,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedRequest;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -57,8 +58,9 @@
 /**
  * Request class to shrink an index into a single shard
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ResizeRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject {
 
     public static final ObjectParser PARSER = new ObjectParser<>("resize_request");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java
index 855e678c77b9b..f9d90d46b0904 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java
@@ -36,14 +36,16 @@
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
 /**
  * Transport request builder for resizing an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ResizeRequestBuilder extends AcknowledgedRequestBuilder {
     public ResizeRequestBuilder(OpenSearchClient client, ActionType action) {
         super(client, action, new ResizeRequest());
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java
index 1aa09023e3583..a4801f84c9ef9 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.shrink;
 
 import org.opensearch.action.admin.indices.create.CreateIndexResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.XContentParser;
@@ -42,8 +43,9 @@
 /**
  * A response for a resize index action, either shrink or split index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ResizeResponse extends CreateIndexResponse {
 
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeType.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeType.java
index 6403ed735ae49..91bcc0d62b1c6 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeType.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeType.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.action.admin.indices.shrink;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * The type of the resize operation
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum ResizeType {
     SHRINK,
     SPLIT,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java
index e4abaef4ddfa8..8bfeb13b253c3 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.store.AlreadyClosedException;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -68,8 +69,9 @@
 /**
  * Common Stats for OpenSearch
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CommonStats implements Writeable, ToXContentFragment {
 
     @Nullable
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java
index 7503020d1c8ef..a7d9f95b80f7b 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.stats;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -45,8 +46,9 @@
 /**
  * Common Stats Flags for OpenSearch
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CommonStatsFlags implements Writeable, Cloneable {
 
     public static final CommonStatsFlags ALL = new CommonStatsFlags().all();
@@ -255,8 +257,9 @@ public CommonStatsFlags clone() {
     /**
      * The flags.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Flag {
         Store("store", 0),
         Indexing("indexing", 1),
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java
index 1635ce0bf83fc..b0143d9491087 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexShardStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.stats;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * IndexShardStats for OpenSearch
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexShardStats implements Iterable, Writeable {
 
     private final ShardId shardId;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexStats.java
index 1c57ca39576b0..09614ea801193 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndexStats.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.action.admin.indices.stats;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -41,8 +43,9 @@
 /**
  * Index Stats for OpenSearch
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexStats implements Iterable {
 
     private final String index;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java
index 54f3e9b7d1a24..2b64464a76899 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.stats;
 
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -46,8 +47,9 @@
  * All the stats to be returned can be cleared using {@link #clear()}, at which point, specific
  * stats can be enabled.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesStatsRequest extends BroadcastRequest {
 
     private CommonStatsFlags flags = new CommonStatsFlags();
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
index c211812b32c48..acc085a96a896 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
@@ -45,8 +46,9 @@
  * All the stats to be returned can be cleared using {@link #clear()}, at which point, specific
  * stats can be enabled.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder<
     IndicesStatsRequest,
     IndicesStatsResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java
index 6f051fa19c99f..6242081cd2371 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.admin.indices.stats.IndexStats.IndexStatsBuilder;
 import org.opensearch.action.support.broadcast.BroadcastResponse;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -54,8 +55,9 @@
 /**
  * Transport response for retrieving indices stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesStatsResponse extends BroadcastResponse {
 
     private ShardStats[] shards;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java
index 77562fa19b319..4ed1ce95b7de2 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -49,8 +50,9 @@
 /**
  * Shard Stats for OpenSearch
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ShardStats implements Writeable, ToXContentFragment {
 
     private ShardRouting shardRouting;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java
index 85524bddc56d8..e93a428b0af26 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -43,8 +44,9 @@
 /**
  * A request to delete an index template.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteIndexTemplateRequest extends ClusterManagerNodeRequest {
 
     private String name;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java
index 4a990b7837120..60771cfa453ae 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java
@@ -34,12 +34,14 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder for deleting an index template
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteIndexTemplateRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     DeleteIndexTemplateRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java
index 18142eb7b787d..fa45efdc53124 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
 /**
  * Request that allows to retrieve index templates
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetIndexTemplatesRequest extends ClusterManagerNodeReadRequest {
 
     private String[] names;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java
index 09de1733239fc..f8c02b4c8be08 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java
@@ -33,12 +33,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Request builder to retrieve one or more Index templates
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetIndexTemplatesRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     GetIndexTemplatesRequest,
     GetIndexTemplatesResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
index 761a345b49538..009a73b615e0c 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
@@ -32,6 +32,7 @@
 package org.opensearch.action.admin.indices.template.get;
 
 import org.opensearch.cluster.metadata.IndexTemplateMetadata;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -49,8 +50,9 @@
 /**
  * Response for retrieving one or more Index templates
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject {
 
     private final List indexTemplates;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java
index 028275778538e..c4396f22d7c16 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -47,8 +48,9 @@
  * Transport Request for handling simulating an index template either by name (looking it up in the
  * cluster state), or by a provided template configuration
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SimulateIndexTemplateRequest extends ClusterManagerNodeReadRequest {
 
     private String indexName;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
index 8a370a6845db6..c6a9d3530a8cc 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.cluster.metadata.Template;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -50,8 +51,9 @@
 /**
  * Contains the information on what V2 templates would match a given index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SimulateIndexTemplateResponse extends ActionResponse implements ToXContentObject {
 
     private static final ParseField TEMPLATE = new ParseField("template");
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java
index 1979cc39e32fb..ed209e18b64ef 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java
@@ -41,6 +41,7 @@
 import org.opensearch.cluster.metadata.ComposableIndexTemplate;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -54,8 +55,9 @@
 /**
  * An action for putting a composable template into the cluster state
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutComposableIndexTemplateAction extends ActionType {
 
     public static final PutComposableIndexTemplateAction INSTANCE = new PutComposableIndexTemplateAction();
@@ -68,8 +70,9 @@ private PutComposableIndexTemplateAction() {
     /**
      * A request for putting a single index template into the cluster state
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Request extends ClusterManagerNodeRequest implements IndicesRequest {
         private final String name;
         @Nullable
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
index 4f7a28a03e6fb..d4e9200508bfa 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
@@ -41,6 +41,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -78,8 +79,9 @@
 /**
  * A request to create an index template.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutIndexTemplateRequest extends ClusterManagerNodeRequest
     implements
         IndicesRequest,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
index 04d2236e00e8f..931d12de574ae 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -47,8 +48,9 @@
 /**
  * A request builder for putting an index template into the cluster state
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutIndexTemplateRequestBuilder extends ClusterManagerNodeOperationRequestBuilder<
     PutIndexTemplateRequest,
     AcknowledgedResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java
index 32cf6d4bfe70d..e760067e9f5be 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexShardUpgradeStatus.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.upgrade.get;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.index.shard.ShardId;
 
 import java.util.Arrays;
@@ -40,8 +41,9 @@
 /**
  * Status for an Index Shard Upgrade
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexShardUpgradeStatus implements Iterable {
 
     private final ShardId shardId;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexUpgradeStatus.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexUpgradeStatus.java
index 4513a321e2a51..fd8ddc1293aaf 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexUpgradeStatus.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/IndexUpgradeStatus.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.action.admin.indices.upgrade.get;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -41,8 +43,9 @@
 /**
  * Status for an Index Upgrade
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexUpgradeStatus implements Iterable {
 
     private final String index;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java
index 57fb2513faf78..783b44ba6570d 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.support.broadcast.BroadcastShardResponse;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -42,8 +43,9 @@
 /**
  * Status for a Shard Upgrade
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ShardUpgradeStatus extends BroadcastShardResponse {
 
     private ShardRouting shardRouting;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java
index 987661d2129ac..2584ab6b370da 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.upgrade.get;
 
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 
@@ -41,8 +42,9 @@
 /**
  * Transport Request for retrieving status of upgrading an Index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpgradeStatusRequest extends BroadcastRequest {
 
     public UpgradeStatusRequest() {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequestBuilder.java
index c698c38fe12d5..ac5f881c35dc5 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport Request Builder for retrieving status of upgrading an Index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpgradeStatusRequestBuilder extends BroadcastOperationRequestBuilder<
     UpgradeStatusRequest,
     UpgradeStatusResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java
index 20dbd12287231..ba2915ee4ddf1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.upgrade.get;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -50,8 +51,9 @@
 /**
  * Transport Response for retrieving status of upgrading an Index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpgradeStatusResponse extends BroadcastResponse {
     private ShardUpgradeStatus[] shards;
 
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java
index 98c307c37ea54..4df02ad7fa8c0 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.upgrade.post;
 
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -45,8 +46,9 @@
  * @see org.opensearch.client.IndicesAdminClient#upgrade(UpgradeRequest)
  * @see UpgradeResponse
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpgradeRequest extends BroadcastRequest {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequestBuilder.java
index 8203f9d51b8e4..bf316504dc920 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A request to upgrade one or more indices. In order to optimize on all the indices, pass an empty array or
  * {@code null} for the indices.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpgradeRequestBuilder extends BroadcastOperationRequestBuilder {
 
     public UpgradeRequestBuilder(OpenSearchClient client, UpgradeAction action) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java
index d08cc65832f8f..4bdf41dabba37 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -47,8 +48,9 @@
 /**
  * A response for the upgrade action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpgradeResponse extends BroadcastResponse {
 
     private final Map> versions;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java
index 248cab5e40eaf..84aa687af0fe1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/QueryExplanation.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.validate.query;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -50,8 +51,9 @@
 /**
  * Query Explanation
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class QueryExplanation implements Writeable, ToXContentFragment {
 
     public static final String INDEX_FIELD = "index";
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java
index ef0d14502af23..94bec696dd2c1 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.ValidateActions;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -53,8 +54,9 @@
  * 
                              
  * The request requires the query to be set using {@link #query(QueryBuilder)}
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ValidateQueryRequest extends BroadcastRequest implements ToXContentObject {
 
     private QueryBuilder query = new MatchAllQueryBuilder();
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java
index 6209f41d88be2..de4cf5ae2b904 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.index.query.QueryBuilder;
 
 /**
  * Transport Request Builder to Validate a Query
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ValidateQueryRequestBuilder extends BroadcastOperationRequestBuilder<
     ValidateQueryRequest,
     ValidateQueryResponse,
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java
index 44d6d637cdc51..791128491f33d 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/validate/query/ValidateQueryResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.validate.query;
 
 import org.opensearch.action.support.broadcast.BroadcastResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.support.DefaultShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -52,8 +53,9 @@
 /**
  * The response of the validate action.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ValidateQueryResponse extends BroadcastResponse {
 
     public static final String VALID_FIELD = "valid";
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java
index 20dea872e0ad2..81e8940f1b505 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkItemResponse.java
@@ -41,6 +41,7 @@
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.action.update.UpdateResponse;
 import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
@@ -68,8 +69,9 @@
  * Represents a single item response for an action executed as part of the bulk API. Holds the index/type/id
  * of the relevant action, and if it has failed or not (with the failure message in case it failed).
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BulkItemResponse implements Writeable, StatusToXContentObject {
 
     private static final String _INDEX = "_index";
@@ -179,8 +181,9 @@ public static BulkItemResponse fromXContent(XContentParser parser, int id) throw
     /**
      * Represents a failure.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Failure implements Writeable, ToXContentFragment {
         public static final String INDEX_FIELD = "index";
         public static final String ID_FIELD = "id";
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
index f2f3077001a13..47abd0337fcf9 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
@@ -45,6 +45,7 @@
 import org.opensearch.action.support.replication.ReplicationRequest;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -71,8 +72,9 @@
  * Note that we only support refresh on the bulk request not per item.
  * @see org.opensearch.client.Client#bulk(BulkRequest)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BulkRequest extends ActionRequest implements CompositeIndicesRequest, WriteRequest, Accountable {
 
     private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(BulkRequest.class);
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java
index 08eee82a53cf9..a165d186d3878 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java
@@ -44,6 +44,7 @@
 import org.opensearch.action.update.UpdateRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.xcontent.MediaType;
@@ -52,8 +53,9 @@
  * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes
  * it in a single batch.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BulkRequestBuilder extends ActionRequestBuilder implements WriteRequestBuilder {
 
     public BulkRequestBuilder(OpenSearchClient client, BulkAction action, @Nullable String globalIndex) {
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java b/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java
index 655e65bc22138..6b70e2acd41d2 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.bulk;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.action.ActionResponse;
@@ -56,8 +57,9 @@
  * bulk requests. Each item holds the index/type/id is operated on, and if it failed or not (with the
  * failure message).
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BulkResponse extends ActionResponse implements Iterable, StatusToXContentObject {
 
     private static final String ITEMS = "items";
diff --git a/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java b/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java
index 9a9e861ad8055..d37e049d44720 100644
--- a/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java
+++ b/server/src/main/java/org/opensearch/action/delete/DeleteRequest.java
@@ -39,6 +39,7 @@
 import org.opensearch.action.DocWriteRequest;
 import org.opensearch.action.support.replication.ReplicatedWriteRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -64,8 +65,9 @@
  * @see org.opensearch.client.Client#delete(DeleteRequest)
  * @see org.opensearch.client.Requests#deleteRequest(String)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteRequest extends ReplicatedWriteRequest
     implements
         DocWriteRequest,
diff --git a/server/src/main/java/org/opensearch/action/delete/DeleteRequestBuilder.java b/server/src/main/java/org/opensearch/action/delete/DeleteRequestBuilder.java
index 0436962ce01d2..66aa1c73042fe 100644
--- a/server/src/main/java/org/opensearch/action/delete/DeleteRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/delete/DeleteRequestBuilder.java
@@ -36,13 +36,15 @@
 import org.opensearch.action.support.replication.ReplicationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.index.VersionType;
 
 /**
  * A delete document action request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteRequestBuilder extends ReplicationRequestBuilder
     implements
         WriteRequestBuilder {
diff --git a/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java b/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java
index e4d44197a8885..c39a787ad763d 100644
--- a/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java
+++ b/server/src/main/java/org/opensearch/action/delete/DeleteResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.delete;
 
 import org.opensearch.action.DocWriteResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.rest.RestStatus;
@@ -48,8 +49,9 @@
  * @see org.opensearch.action.delete.DeleteRequest
  * @see org.opensearch.client.Client#delete(DeleteRequest)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeleteResponse extends DocWriteResponse {
 
     public DeleteResponse(ShardId shardId, StreamInput in) throws IOException {
@@ -112,8 +114,9 @@ public static void parseXContentFields(XContentParser parser, Builder context) t
      * temporarily store the parsed values, then the {@link DocWriteResponse.Builder#build()} method is called to
      * instantiate the {@link DeleteResponse}.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder extends DocWriteResponse.Builder {
 
         @Override
diff --git a/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java b/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java
index aad5a9853a3c5..dd9c01b04acbb 100644
--- a/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java
+++ b/server/src/main/java/org/opensearch/action/explain/ExplainRequest.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.ValidateActions;
 import org.opensearch.action.support.single.shard.SingleShardRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -54,8 +55,9 @@
 /**
  * Explain request encapsulating the explain query and document identifier to get an explanation for.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ExplainRequest extends SingleShardRequest implements ToXContentObject {
 
     private static final ParseField QUERY_FIELD = new ParseField("query");
diff --git a/server/src/main/java/org/opensearch/action/explain/ExplainRequestBuilder.java b/server/src/main/java/org/opensearch/action/explain/ExplainRequestBuilder.java
index 3031cb6067469..681b48f7a6593 100644
--- a/server/src/main/java/org/opensearch/action/explain/ExplainRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/explain/ExplainRequestBuilder.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.single.shard.SingleShardOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.search.fetch.subphase.FetchSourceContext;
@@ -42,8 +43,9 @@
 /**
  * A builder for {@link ExplainRequest}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ExplainRequestBuilder extends SingleShardOperationRequestBuilder {
 
     ExplainRequestBuilder(OpenSearchClient client, ExplainAction action) {
diff --git a/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java b/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java
index 895f87b83b6ff..80a8634e62d87 100644
--- a/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java
+++ b/server/src/main/java/org/opensearch/action/explain/ExplainResponse.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.search.Explanation;
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -56,8 +57,9 @@
 /**
  * Response containing the score explanation.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ExplainResponse extends ActionResponse implements StatusToXContentObject {
 
     private static final ParseField _INDEX = new ParseField("_index");
diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java
index 9b01f65f0f711..20b9789972fff 100644
--- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java
+++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilities.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.fieldcaps;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -60,8 +61,9 @@
 /**
  * Describes the capabilities of a field optionally merged across multiple indices.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FieldCapabilities implements Writeable, ToXContentObject {
 
     private static final ParseField TYPE_FIELD = new ParseField("type");
diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java
index c1c54c7018bcf..71e8db73e4ecc 100644
--- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java
+++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.ValidateActions;
 import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -52,7 +53,10 @@
 
 /**
  * Transport request for retrieving field capabilities for an explicit list of fields
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class FieldCapabilitiesRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContentObject {
     public static final String NAME = "field_caps_request";
 
diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java
index 70a90b98bdf25..c589d344089f3 100644
--- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java
@@ -34,13 +34,15 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.index.query.QueryBuilder;
 
 /**
  * Transport request builder for retrieving field capabilities
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FieldCapabilitiesRequestBuilder extends ActionRequestBuilder {
     public FieldCapabilitiesRequestBuilder(OpenSearchClient client, FieldCapabilitiesAction action, String... indices) {
         super(client, action, new FieldCapabilitiesRequest().indices(indices));
diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java
index 2dcda2bf56b8f..72fdc75686e3b 100644
--- a/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java
+++ b/server/src/main/java/org/opensearch/action/fieldcaps/FieldCapabilitiesResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.fieldcaps;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -57,8 +58,10 @@
 /**
  * Response for {@link FieldCapabilitiesRequest} requests.
  *
- * @opensearch.internal
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FieldCapabilitiesResponse extends ActionResponse implements ToXContentObject {
     private static final ParseField INDICES_FIELD = new ParseField("indices");
     private static final ParseField FIELDS_FIELD = new ParseField("fields");
diff --git a/server/src/main/java/org/opensearch/action/get/GetRequest.java b/server/src/main/java/org/opensearch/action/get/GetRequest.java
index 62d85d6d8f352..952fa8bdab63a 100644
--- a/server/src/main/java/org/opensearch/action/get/GetRequest.java
+++ b/server/src/main/java/org/opensearch/action/get/GetRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.RealtimeRequest;
 import org.opensearch.action.ValidateActions;
 import org.opensearch.action.support.single.shard.SingleShardRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -60,8 +61,9 @@
  * @see org.opensearch.client.Requests#getRequest(String)
  * @see org.opensearch.client.Client#get(GetRequest)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetRequest extends SingleShardRequest implements RealtimeRequest {
 
     private String id;
diff --git a/server/src/main/java/org/opensearch/action/get/GetRequestBuilder.java b/server/src/main/java/org/opensearch/action/get/GetRequestBuilder.java
index 6237cf73f0ca8..f50cbb16186f7 100644
--- a/server/src/main/java/org/opensearch/action/get/GetRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/get/GetRequestBuilder.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.support.single.shard.SingleShardOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.index.VersionType;
 import org.opensearch.search.fetch.subphase.FetchSourceContext;
@@ -42,8 +43,9 @@
 /**
  * A get document action request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetRequestBuilder extends SingleShardOperationRequestBuilder {
 
     public GetRequestBuilder(OpenSearchClient client, GetAction action) {
diff --git a/server/src/main/java/org/opensearch/action/get/GetResponse.java b/server/src/main/java/org/opensearch/action/get/GetResponse.java
index c86128444d7eb..f7f7241933bd6 100644
--- a/server/src/main/java/org/opensearch/action/get/GetResponse.java
+++ b/server/src/main/java/org/opensearch/action/get/GetResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.get;
 
 import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.document.DocumentField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.ParsingException;
@@ -58,8 +59,9 @@
  * @see GetRequest
  * @see org.opensearch.client.Client#get(GetRequest)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetResponse extends ActionResponse implements Iterable, ToXContentObject {
 
     GetResult getResult;
diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java b/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java
index 19c9b785e7ea2..09b4205ffe521 100644
--- a/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java
+++ b/server/src/main/java/org/opensearch/action/get/MultiGetItemResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.get;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -41,8 +42,9 @@
 /**
  * A single multi get response.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiGetItemResponse implements Writeable {
 
     private final GetResponse response;
diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java b/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java
index 7f0844d717488..b15c69a41972f 100644
--- a/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java
+++ b/server/src/main/java/org/opensearch/action/get/MultiGetRequest.java
@@ -42,6 +42,7 @@
 import org.opensearch.action.ValidateActions;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.ParsingException;
@@ -69,8 +70,9 @@
 /**
  * Transport request for a multi get.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiGetRequest extends ActionRequest
     implements
         Iterable,
@@ -91,8 +93,9 @@ public class MultiGetRequest extends ActionRequest
     /**
      * A single get item.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Item implements Writeable, IndicesRequest, ToXContentObject {
 
         private String index;
diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetRequestBuilder.java b/server/src/main/java/org/opensearch/action/get/MultiGetRequestBuilder.java
index c317edc07da8b..0b701c8ec11c7 100644
--- a/server/src/main/java/org/opensearch/action/get/MultiGetRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/get/MultiGetRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A multi get document action request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiGetRequestBuilder extends ActionRequestBuilder {
 
     public MultiGetRequestBuilder(OpenSearchClient client, MultiGetAction action) {
diff --git a/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java b/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java
index 73372b85be3b2..3c3489a065d9a 100644
--- a/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java
+++ b/server/src/main/java/org/opensearch/action/get/MultiGetResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -55,8 +56,9 @@
 /**
  * Transport response for a multi get.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiGetResponse extends ActionResponse implements Iterable, ToXContentObject {
 
     private static final ParseField INDEX = new ParseField("_index");
@@ -71,8 +73,9 @@ public class MultiGetResponse extends ActionResponse implements Iterable implements DocWriteRequest, CompositeIndicesRequest {
 
     private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IndexRequest.class);
diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java
index 4bf7634dcb7e1..9d4ad3c32778c 100644
--- a/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/index/IndexRequestBuilder.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.support.replication.ReplicationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -47,8 +48,9 @@
 /**
  * An index document action request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexRequestBuilder extends ReplicationRequestBuilder
     implements
         WriteRequestBuilder {
diff --git a/server/src/main/java/org/opensearch/action/index/IndexResponse.java b/server/src/main/java/org/opensearch/action/index/IndexResponse.java
index d0aa9b57b6528..53f832fc12c43 100644
--- a/server/src/main/java/org/opensearch/action/index/IndexResponse.java
+++ b/server/src/main/java/org/opensearch/action/index/IndexResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.index;
 
 import org.opensearch.action.DocWriteResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.index.shard.ShardId;
@@ -50,8 +51,9 @@
  * @see IndexRequest
  * @see org.opensearch.client.Client#index(IndexRequest)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexResponse extends DocWriteResponse {
 
     public IndexResponse(ShardId shardId, StreamInput in) throws IOException {
@@ -116,8 +118,9 @@ public static void parseXContentFields(XContentParser parser, Builder context) t
      * temporarily store the parsed values, then the {@link Builder#build()} method is called to
      * instantiate the {@link IndexResponse}.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder extends DocWriteResponse.Builder {
         @Override
         public IndexResponse build() {
diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java
index 2f05ce3a25320..b9d916e152c3d 100644
--- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -43,8 +44,9 @@
 /**
  * transport request to delete a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeletePipelineRequest extends AcknowledgedRequest {
 
     private String id;
diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java
index 6a2eb494e8d3f..bc253db85bb0f 100644
--- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder to delete a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeletePipelineRequestBuilder extends ActionRequestBuilder {
 
     public DeletePipelineRequestBuilder(OpenSearchClient client, DeletePipelineAction action) {
diff --git a/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java
index 4bae98098777d..c7266c31a5022 100644
--- a/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -43,8 +44,9 @@
 /**
  * transport request to get a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetPipelineRequest extends ClusterManagerNodeReadRequest {
 
     private String[] ids;
diff --git a/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequestBuilder.java
index bdc13523ffdc6..593ea2156d5e3 100644
--- a/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/ingest/GetPipelineRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * Transport request builder to get a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetPipelineRequestBuilder extends ClusterManagerNodeReadOperationRequestBuilder<
     GetPipelineRequest,
     GetPipelineResponse,
diff --git a/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java b/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java
index bd1b18be9a828..0719842bc985f 100644
--- a/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java
+++ b/server/src/main/java/org/opensearch/action/ingest/GetPipelineResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.ingest;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
@@ -57,8 +58,9 @@
 /**
  * transport response for getting a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetPipelineResponse extends ActionResponse implements StatusToXContentObject {
 
     private List pipelines;
diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java
index f764e4b23860a..06e89b5f2908b 100644
--- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -49,8 +50,9 @@
 /**
  * transport request to put a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutPipelineRequest extends AcknowledgedRequest implements ToXContentObject {
 
     private String id;
diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java
index e734abb6d7969..e8d6a4d332319 100644
--- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java
@@ -35,14 +35,16 @@
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaType;
 
 /**
  * Transport request builder to put a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PutPipelineRequestBuilder extends ActionRequestBuilder {
 
     public PutPipelineRequestBuilder(OpenSearchClient client, PutPipelineAction action) {
diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java
index 98a03272aff42..bc338a57f762d 100644
--- a/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java
+++ b/server/src/main/java/org/opensearch/action/ingest/SimulateDocumentResult.java
@@ -31,14 +31,16 @@
 
 package org.opensearch.action.ingest;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.xcontent.ToXContentObject;
 
 /**
  * Interface to simulate a document result
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface SimulateDocumentResult extends Writeable, ToXContentObject {
 
 }
diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java
index ec3ee981b646f..b51f25d2e62b1 100644
--- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -60,8 +61,9 @@
 /**
  * transport request to simulate a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SimulatePipelineRequest extends ActionRequest implements ToXContentObject {
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(SimulatePipelineRequest.class);
diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java
index 55e6d95fde65c..2a5f281a5075c 100644
--- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequestBuilder.java
@@ -34,14 +34,16 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaType;
 
 /**
  * Transport request builder to simulate a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SimulatePipelineRequestBuilder extends ActionRequestBuilder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java
index 3cbbc4350c3bd..1fb3a69bd1e16 100644
--- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java
+++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.ingest;
 
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -54,8 +55,9 @@
 /**
  * transport response for simulating a pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SimulatePipelineResponse extends ActionResponse implements ToXContentObject {
     private String pipelineId;
     private boolean verbose;
diff --git a/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java b/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java
index cd9e33634d918..16bf0fc46c4eb 100644
--- a/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/ClearScrollRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentObject;
@@ -50,8 +51,9 @@
 /**
  * Transport request for clearing a search scroll
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClearScrollRequest extends ActionRequest implements ToXContentObject {
 
     private List scrollIds;
diff --git a/server/src/main/java/org/opensearch/action/search/ClearScrollRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/ClearScrollRequestBuilder.java
index 63f64e02a9dd2..c10417e756dd6 100644
--- a/server/src/main/java/org/opensearch/action/search/ClearScrollRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/search/ClearScrollRequestBuilder.java
@@ -34,14 +34,16 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.List;
 
 /**
  * Transport request builder for clearing a search scroll
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClearScrollRequestBuilder extends ActionRequestBuilder {
 
     public ClearScrollRequestBuilder(OpenSearchClient client, ClearScrollAction action) {
diff --git a/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java b/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java
index 4428f693763ed..e2580ccdd0969 100644
--- a/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/ClearScrollResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -52,8 +53,9 @@
 /**
  * Transport response for clearing a search scroll
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClearScrollResponse extends ActionResponse implements StatusToXContentObject {
 
     private static final ParseField SUCCEEDED = new ParseField("succeeded");
diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java b/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java
index a47127f18e18d..840d4becda714 100644
--- a/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java
@@ -13,6 +13,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -30,7 +31,10 @@
 
 /**
  * A request to make create point in time against one or more indices.
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class CreatePitRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent {
 
     // keep alive for pit reader context
diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java b/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java
index aa841a02f1d20..410b93afc3e65 100644
--- a/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/CreatePitResponse.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -26,7 +27,10 @@
 
 /**
  * Create point in time response with point in time id and shard success / failures
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class CreatePitResponse extends ActionResponse implements StatusToXContentObject {
     private static final ParseField ID = new ParseField("pit_id");
     private static final ParseField CREATION_TIME = new ParseField("creation_time");
diff --git a/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java b/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java
index 40098c6670b65..c534b306b1404 100644
--- a/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java
+++ b/server/src/main/java/org/opensearch/action/search/DeletePitInfo.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -23,7 +24,10 @@
 
 /**
  * This class captures if deletion of pit is successful along with pit id
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class DeletePitInfo extends TransportResponse implements Writeable, ToXContent {
     /**
      * This will be true if PIT reader contexts are deleted ond also if contexts are not found.
diff --git a/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java b/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java
index 1a66311cd9a1b..e21a63eef9433 100644
--- a/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/DeletePitRequest.java
@@ -11,6 +11,7 @@
 
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContent;
@@ -27,7 +28,10 @@
 
 /**
  * Request to delete one or more PIT search contexts based on IDs.
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class DeletePitRequest extends ActionRequest implements ToXContentObject {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java b/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java
index 00274f0e610a9..469d78ad73bd9 100644
--- a/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/DeletePitResponse.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -28,7 +29,10 @@
 
 /**
  * Response class for delete pits flow which clears the point in time search contexts
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class DeletePitResponse extends ActionResponse implements StatusToXContentObject {
 
     private final List deletePitResults;
diff --git a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java
index 2c6ab6437fd2a..0cde81203063c 100644
--- a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -19,8 +20,9 @@
 /**
  * Request to delete a search pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class DeleteSearchPipelineRequest extends AcknowledgedRequest {
     private String id;
 
diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java
index 9bbc81c0d6a4c..91353240a8156 100644
--- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodeResponse.java
@@ -11,6 +11,7 @@
 
 import org.opensearch.action.support.nodes.BaseNodeResponse;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentFragment;
@@ -22,7 +23,10 @@
 
 /**
  * Inner node get all pits response
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class GetAllPitNodeResponse extends BaseNodeResponse implements ToXContentFragment {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java
index 948fe72eae817..336c8139561e9 100644
--- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.support.nodes.BaseNodesRequest;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -18,7 +19,10 @@
 
 /**
  * Request to get all active PIT IDs from all nodes of cluster
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class GetAllPitNodesRequest extends BaseNodesRequest {
 
     @Inject
diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java
index 9bb3ab6407696..8d858a00c409b 100644
--- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesResponse.java
@@ -11,6 +11,7 @@
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.nodes.BaseNodesResponse;
 import org.opensearch.cluster.ClusterName;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -31,7 +32,10 @@
 
 /**
  * This class transforms active PIT objects from all nodes to unique PIT objects
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class GetAllPitNodesResponse extends BaseNodesResponse implements ToXContentObject {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java
index f573a37fa5dab..59e95aa87985a 100644
--- a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineRequest.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -20,8 +21,9 @@
 /**
  * Request to get search pipelines
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class GetSearchPipelineRequest extends ClusterManagerNodeReadRequest {
     private final String[] ids;
 
diff --git a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java
index 64dde763e9ff8..0379046c8275d 100644
--- a/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/GetSearchPipelineResponse.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.Strings;
@@ -32,8 +33,9 @@
 /**
  * transport response for getting a search pipeline
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class GetSearchPipelineResponse extends ActionResponse implements StatusToXContentObject {
 
     private final List pipelines;
diff --git a/server/src/main/java/org/opensearch/action/search/ListPitInfo.java b/server/src/main/java/org/opensearch/action/search/ListPitInfo.java
index 220b7247517b9..7e4ed186dd665 100644
--- a/server/src/main/java/org/opensearch/action/search/ListPitInfo.java
+++ b/server/src/main/java/org/opensearch/action/search/ListPitInfo.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -23,7 +24,10 @@
 
 /**
  * This holds information about pit reader context such as pit id and creation time
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class ListPitInfo implements ToXContentFragment, Writeable {
     private final String pitId;
     private final long creationTime;
diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
index 00e0345062d1c..5f46e0c298de4 100644
--- a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java
@@ -38,6 +38,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.IndicesOptions.WildcardStates;
 import org.opensearch.common.CheckedBiConsumer;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -71,8 +72,9 @@
 /**
  * A multi search API request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiSearchRequest extends ActionRequest implements CompositeIndicesRequest {
 
     public static final int MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT = 0;
diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/MultiSearchRequestBuilder.java
index f9bb90c69d925..3bf078e8ce793 100644
--- a/server/src/main/java/org/opensearch/action/search/MultiSearchRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/search/MultiSearchRequestBuilder.java
@@ -35,12 +35,14 @@
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A request builder for multiple search requests.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiSearchRequestBuilder extends ActionRequestBuilder {
 
     public MultiSearchRequestBuilder(OpenSearchClient client, MultiSearchAction action) {
diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java b/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java
index 24ce4c2406570..70bb0b99e69df 100644
--- a/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/MultiSearchResponse.java
@@ -35,6 +35,7 @@
 import org.opensearch.ExceptionsHelper;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
@@ -59,8 +60,9 @@
 /**
  * A multi search response.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiSearchResponse extends ActionResponse implements Iterable, ToXContentObject {
 
     private static final ParseField RESPONSES = new ParseField(Fields.RESPONSES);
@@ -78,8 +80,9 @@ public class MultiSearchResponse extends ActionResponse implements Iterable implements ToXContentObject {
     private String id;
     private BytesReference source;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java
index 4c0fe3ac06326..8cf92934c8a52 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java
@@ -8,10 +8,14 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Enum for different Search Phases in OpenSearch
- * @opensearch.internal
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.9.0")
 public enum SearchPhaseName {
     DFS_PRE_QUERY("dfs_pre_query"),
     QUERY("query"),
diff --git a/server/src/main/java/org/opensearch/action/search/SearchProgressListener.java b/server/src/main/java/org/opensearch/action/search/SearchProgressListener.java
index ffc64682cb07d..34e8aacbad250 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchProgressListener.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchProgressListener.java
@@ -38,6 +38,7 @@
 import org.apache.lucene.search.TotalHits;
 import org.opensearch.action.search.SearchResponse.Clusters;
 import org.opensearch.cluster.routing.GroupShardsIterator;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.search.SearchPhaseResult;
 import org.opensearch.search.SearchShardTarget;
 import org.opensearch.search.aggregations.InternalAggregations;
@@ -53,6 +54,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class SearchProgressListener {
     private static final Logger logger = LogManager.getLogger(SearchProgressListener.class);
 
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
index fb026dae630b7..96cea17ff4972 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
@@ -38,6 +38,7 @@
 import org.opensearch.action.IndicesRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -69,8 +70,9 @@
  * @see org.opensearch.client.Client#search(SearchRequest)
  * @see SearchResponse
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable {
 
     public static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
index bc43b65e5d844..e949c5e0bea29 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.script.Script;
@@ -58,8 +59,9 @@
 /**
  * A search action request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchRequestBuilder extends ActionRequestBuilder {
 
     public SearchRequestBuilder(OpenSearchClient client, SearchAction action) {
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java
index ad299c11b987d..6b7c94ec3037a 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.metrics.MeanMetric;
@@ -19,8 +20,9 @@
 /**
  * Request level search stats to track coordinator level node search latencies
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.11.0")
 public final class SearchRequestStats implements SearchRequestOperationsListener {
     Map phaseStatsMap = new EnumMap<>(SearchPhaseName.class);
 
diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponse.java b/server/src/main/java/org/opensearch/action/search/SearchResponse.java
index 96d07982d03db..899c71e91e3ab 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchResponse.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.search.TotalHits;
 import org.opensearch.Version;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.StatusToXContentObject;
 import org.opensearch.core.ParseField;
@@ -77,8 +78,9 @@
 /**
  * A response of a search request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchResponse extends ActionResponse implements StatusToXContentObject {
 
     private static final ParseField SCROLL_ID = new ParseField("_scroll_id");
@@ -573,8 +575,9 @@ public String toString() {
      * Holds info about the clusters that the search was executed on: how many in total, how many of them were successful
      * and how many of them were skipped.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Clusters implements ToXContentFragment, Writeable {
 
         public static final Clusters EMPTY = new Clusters(0, 0, 0);
@@ -677,8 +680,9 @@ public String toString() {
      * Holds info about the clusters that the search was executed on: how many in total, how many of them were successful
      * and how many of them were skipped.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class PhaseTook implements ToXContentFragment, Writeable {
         static final ParseField PHASE_TOOK = new ParseField("phase_took");
         private final Map phaseTookMap;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java
index 579a01f0dd932..bca2c8a52b691 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentFragment;
@@ -58,8 +59,9 @@
  * to parse aggregations into, which are not serializable. This is the common part that can be
  * shared between core and client.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchResponseSections implements ToXContentFragment {
 
     public static final ParseField EXT_FIELD = new ParseField("ext");
diff --git a/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java b/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java
index cda00811e5500..044efdc36d04f 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchScrollRequest.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -53,8 +54,9 @@
 /**
  * Transport request for a search scroll
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchScrollRequest extends ActionRequest implements ToXContentObject {
 
     private String scrollId;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchScrollRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/SearchScrollRequestBuilder.java
index 638c595216631..41b34bd9c6c9e 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchScrollRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchScrollRequestBuilder.java
@@ -34,14 +34,16 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.search.Scroll;
 
 /**
  * A search scroll action request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchScrollRequestBuilder extends ActionRequestBuilder {
 
     public SearchScrollRequestBuilder(OpenSearchClient client, SearchScrollAction action) {
diff --git a/server/src/main/java/org/opensearch/action/search/SearchTask.java b/server/src/main/java/org/opensearch/action/search/SearchTask.java
index 20370b7b17a07..d3c1043c50cce 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchTask.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchTask.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.tasks.TaskId;
 import org.opensearch.tasks.CancellableTask;
@@ -45,8 +46,9 @@
 /**
  * Task storing information about a currently running {@link SearchRequest}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchTask extends CancellableTask implements SearchBackpressureTask {
     // generating description in a lazy way since source can be quite big
     private final Supplier descriptionSupplier;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchType.java b/server/src/main/java/org/opensearch/action/search/SearchType.java
index cb86c0d6c1b4a..e549ec598380a 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchType.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchType.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Search type represent the manner at which the search operation is executed.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum SearchType {
     /**
      * Same as {@link #QUERY_THEN_FETCH}, except for an initial scatter phase which goes and computes the distributed
diff --git a/server/src/main/java/org/opensearch/action/support/IndicesOptions.java b/server/src/main/java/org/opensearch/action/support/IndicesOptions.java
index d30ee7e11bdfa..2d9fecddb6f7d 100644
--- a/server/src/main/java/org/opensearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/opensearch/action/support/IndicesOptions.java
@@ -32,6 +32,7 @@
 package org.opensearch.action.support;
 
 import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -55,15 +56,17 @@
  * Controls how to deal with unavailable concrete indices (closed or missing), how wildcard expressions are expanded
  * to actual indices (all, closed or open indices) and how to deal with wildcard expressions that resolve to no indices.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndicesOptions implements ToXContentFragment {
 
     /**
      * The wildcard states.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum WildcardStates {
         OPEN,
         CLOSED,
diff --git a/server/src/main/java/org/opensearch/action/support/WriteRequest.java b/server/src/main/java/org/opensearch/action/support/WriteRequest.java
index f462464b99ce8..8d53f7b005d54 100644
--- a/server/src/main/java/org/opensearch/action/support/WriteRequest.java
+++ b/server/src/main/java/org/opensearch/action/support/WriteRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.support.replication.ReplicatedWriteRequest;
 import org.opensearch.action.update.UpdateRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -78,8 +79,9 @@ default R setRefreshPolicy(String refreshPolicy) {
     /**
      * The refresh policy of the request.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     enum RefreshPolicy implements Writeable {
         /**
          * Don't refresh after this request. The default.
diff --git a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java
index 5d09d880336f1..279ad401f7e56 100644
--- a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java
+++ b/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.action.support.master;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -49,8 +50,9 @@
 /**
  * A response that indicates that a request has been acknowledged
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AcknowledgedResponse extends ActionResponse implements ToXContentObject {
 
     private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged");
diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java
index 80ca1629417ad..8dbf3dd4df512 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsItemResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.termvectors;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -41,8 +42,9 @@
 /**
  * A single multi term response.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiTermVectorsItemResponse implements Writeable {
 
     private final TermVectorsResponse response;
diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java
index c055564c3fcbe..0eef737a54bb3 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequest.java
@@ -39,6 +39,7 @@
 import org.opensearch.action.RealtimeRequest;
 import org.opensearch.action.ValidateActions;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.XContentParser;
@@ -54,8 +55,9 @@
 /**
  * A single multi get request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiTermVectorsRequest extends ActionRequest
     implements
         Iterable,
diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequestBuilder.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequestBuilder.java
index 04dfd39112d6e..6bfa402575885 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsRequestBuilder.java
@@ -34,12 +34,14 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A single multi get request builder.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiTermVectorsRequestBuilder extends ActionRequestBuilder {
 
     public MultiTermVectorsRequestBuilder(OpenSearchClient client, MultiTermVectorsAction action) {
diff --git a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java
index 5d40e64df1e3e..8a059829dda0e 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/MultiTermVectorsResponse.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -48,15 +49,17 @@
 /**
  * A multi get response.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MultiTermVectorsResponse extends ActionResponse implements Iterable, ToXContentObject {
 
     /**
      * Represents a failure.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Failure implements Writeable {
         private final String index;
         private final String id;
diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFilter.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFilter.java
index 0e0202777794b..89d2f8567b3cb 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFilter.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsFilter.java
@@ -39,6 +39,7 @@
 import org.apache.lucene.search.similarities.ClassicSimilarity;
 import org.apache.lucene.search.similarities.TFIDFSimilarity;
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -49,8 +50,9 @@
  * Filter the term vector (doc frequency, positions, offsets) for a
  * document.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TermVectorsFilter {
     public static final int DEFAULT_MAX_QUERY_TERMS = 25;
     public static final int DEFAULT_MIN_TERM_FREQ = 0;
@@ -179,8 +181,9 @@ public void setMaxWordLength(int maxWordLength) {
     /**
      * Internal score term
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static final class ScoreTerm {
         public String field;
         public String word;
diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java
index 825b0b4982880..a761cabb9599a 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java
@@ -40,6 +40,7 @@
 import org.opensearch.action.get.MultiGetRequest;
 import org.opensearch.action.support.single.shard.SingleShardRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.common.xcontent.XContentType;
@@ -74,8 +75,9 @@
  * Note, the {@link #index()}, and {@link #id(String)} are
  * required.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TermVectorsRequest extends SingleShardRequest implements RealtimeRequest {
     private static final ParseField INDEX = new ParseField("_index");
     private static final ParseField ID = new ParseField("_id");
@@ -118,8 +120,9 @@ public class TermVectorsRequest extends SingleShardRequest i
     /**
      * Internal filter settings
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static final class FilterSettings {
         public Integer maxNumTerms;
         public Integer minTermFreq;
@@ -572,8 +575,9 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * The flags.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Flag {
         // Do not change the order of these flags we use
         // the ordinal for encoding! Only append to the end!
diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequestBuilder.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequestBuilder.java
index 02cfff1a6682b..ce68c06206fef 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequestBuilder.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.ActionRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.VersionType;
 
@@ -46,8 +47,9 @@
  * Note, the {@code index}, {@code type} and {@code id} are
  * required.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TermVectorsRequestBuilder extends ActionRequestBuilder {
 
     public TermVectorsRequestBuilder(OpenSearchClient client, TermVectorsAction action) {
diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java
index 3c338ce7338bb..7ad27808588ae 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsResponse.java
@@ -41,6 +41,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionResponse;
@@ -62,8 +63,9 @@
  * Response returning the term vector (doc frequency, positions, offsets) for a
  * document.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TermVectorsResponse extends ActionResponse implements ToXContentObject {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
index 946c3c2446173..9654bd1c114ba 100644
--- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
+++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
@@ -42,6 +42,7 @@
 import org.opensearch.action.support.replication.ReplicationRequest;
 import org.opensearch.action.support.single.instance.InstanceShardOperationRequest;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.XContentHelper;
@@ -74,8 +75,9 @@
 /**
  * Transport request for updating an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpdateRequest extends InstanceShardOperationRequest
     implements
         DocWriteRequest,
diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequestBuilder.java b/server/src/main/java/org/opensearch/action/update/UpdateRequestBuilder.java
index c97d0b4f5d13d..d662381ac5e19 100644
--- a/server/src/main/java/org/opensearch/action/update/UpdateRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/update/UpdateRequestBuilder.java
@@ -39,6 +39,7 @@
 import org.opensearch.action.support.single.instance.InstanceShardOperationRequestBuilder;
 import org.opensearch.client.OpenSearchClient;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.VersionType;
@@ -49,8 +50,9 @@
 /**
  * Transport request builder for updating an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder
     implements
         WriteRequestBuilder {
diff --git a/server/src/main/java/org/opensearch/action/update/UpdateResponse.java b/server/src/main/java/org/opensearch/action/update/UpdateResponse.java
index d9f6ceacb0f3b..c7ee1742af0f2 100644
--- a/server/src/main/java/org/opensearch/action/update/UpdateResponse.java
+++ b/server/src/main/java/org/opensearch/action/update/UpdateResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.update;
 
 import org.opensearch.action.DocWriteResponse;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.index.shard.ShardId;
@@ -48,8 +49,9 @@
 /**
  * Transport response for updating an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class UpdateResponse extends DocWriteResponse {
 
     private static final String GET = "get";
@@ -173,8 +175,9 @@ public static void parseXContentFields(XContentParser parser, Builder context) t
      * temporarily store the parsed values, then the {@link DocWriteResponse.Builder#build()} method is called to
      * instantiate the {@link UpdateResponse}.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder extends DocWriteResponse.Builder {
 
         private GetResult getResult = null;
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java b/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java
index fab104142e5bb..904083e96032f 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterChangedEvent.java
@@ -37,6 +37,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.index.Index;
 import org.opensearch.gateway.GatewayService;
 
@@ -52,8 +53,9 @@
 /**
  * An event received by the local node, signaling that the cluster state has changed.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterChangedEvent {
 
     private final String source;
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java
index 5bd858a53bee2..4c38d6fd99f5d 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -56,8 +57,9 @@
  * InternalClusterInfoService.shardIdentifierFromRouting(String)
  * for the key used in the shardSizes map
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterInfo implements ToXContentFragment, Writeable {
     private final Map leastAvailableSpaceUsage;
     private final Map mostAvailableSpaceUsage;
@@ -287,8 +289,9 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Represents the total amount of "reserved" space on a particular data path, together with the set of shards considered.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ReservedSpace implements Writeable {
 
         public static final ReservedSpace EMPTY = new ReservedSpace(0, new HashSet<>());
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterName.java b/server/src/main/java/org/opensearch/cluster/ClusterName.java
index 44321f7de9395..d6149421c3fd0 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterName.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterName.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.cluster;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -45,8 +46,9 @@
 /**
  * Cluster Name
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterName implements Writeable {
 
     public static final Setting CLUSTER_NAME_SETTING = new Setting<>("cluster.name", "opensearch", (s) -> {
diff --git a/server/src/main/java/org/opensearch/cluster/Diff.java b/server/src/main/java/org/opensearch/cluster/Diff.java
index c0e8e7038d9b4..77301b7c04b66 100644
--- a/server/src/main/java/org/opensearch/cluster/Diff.java
+++ b/server/src/main/java/org/opensearch/cluster/Diff.java
@@ -32,13 +32,15 @@
 
 package org.opensearch.cluster;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.Writeable;
 
 /**
  * Represents difference between states of cluster state parts
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Diff extends Writeable {
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/DiskUsage.java b/server/src/main/java/org/opensearch/cluster/DiskUsage.java
index c472522baee51..33ed030c58a02 100644
--- a/server/src/main/java/org/opensearch/cluster/DiskUsage.java
+++ b/server/src/main/java/org/opensearch/cluster/DiskUsage.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.cluster;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
 /**
  * Encapsulation class used to represent the amount of disk used on a node.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DiskUsage implements ToXContentFragment, Writeable {
     final String nodeId;
     final String nodeName;
diff --git a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java
index eae79e2ac0986..b4d797a39dd2f 100644
--- a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java
+++ b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributeValueHealth.java
@@ -14,6 +14,7 @@
 import org.opensearch.cluster.routing.RoutingNode;
 import org.opensearch.cluster.routing.ShardRoutingState;
 import org.opensearch.cluster.routing.WeightedRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -31,7 +32,10 @@
 
 /**
  * Cluster Awareness AttributeValue Health information
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterAwarenessAttributeValueHealth implements Writeable, ToXContentFragment {
 
     private static final String ACTIVE_SHARDS = "active_shards";
diff --git a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java
index 01d9cfc4438e3..3de2260d0e8bd 100644
--- a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java
+++ b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessAttributesHealth.java
@@ -11,6 +11,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -34,7 +35,9 @@
 /**
  * Cluster Awareness health information
  *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterAwarenessAttributesHealth implements Iterable, Writeable, ToXContentFragment {
 
     private final String awarenessAttributeName;
diff --git a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java
index 1643db5a1f460..841764110626d 100644
--- a/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java
+++ b/server/src/main/java/org/opensearch/cluster/awarenesshealth/ClusterAwarenessHealth.java
@@ -11,6 +11,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance;
 import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -30,7 +31,10 @@
 
 /**
  * Cluster state Awareness health information
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterAwarenessHealth implements Writeable, ToXContentFragment, Iterable {
 
     private static final String AWARENESS_ATTRIBUTE = "awareness_attributes";
diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java
index 40a5080ba74e7..5fa897c0b1185 100644
--- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java
+++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java
@@ -33,6 +33,7 @@
 package org.opensearch.cluster.block;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -48,8 +49,9 @@
 /**
  * Blocks the cluster for concurrency
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterBlock implements Writeable, ToXContentFragment {
 
     private final int id;
diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java
index 5ec847e100c86..5d3bf94aedb19 100644
--- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java
+++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java
@@ -32,13 +32,16 @@
 
 package org.opensearch.cluster.block;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.EnumSet;
 
 /**
  * What level to block the cluster
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum ClusterBlockLevel {
     READ,
     WRITE,
diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java
index 3f67870781580..914743299b023 100644
--- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java
+++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttribute.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.cluster.decommission;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -18,8 +19,9 @@
 /**
  * {@link DecommissionAttribute} encapsulates information about decommissioned node attribute like attribute name, attribute value.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public final class DecommissionAttribute implements Writeable {
     private final String attributeName;
     private final String attributeValue;
diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionStatus.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionStatus.java
index 4ca8c3cc4286e..f4f6cbf632ae2 100644
--- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionStatus.java
+++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionStatus.java
@@ -8,9 +8,14 @@
 
 package org.opensearch.cluster.decommission;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * An enumeration of the states during decommissioning
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public enum DecommissionStatus {
     /**
      * Decommission process is initiated, and to-be-decommissioned leader is excluded from voting config
diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java b/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java
index 5ea482b8b8ffa..06f6b3d57385d 100644
--- a/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java
+++ b/server/src/main/java/org/opensearch/cluster/health/ClusterHealthStatus.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.cluster.health;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -41,8 +42,9 @@
 /**
  * Cluster health status
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum ClusterHealthStatus implements Writeable {
     GREEN((byte) 0),
     YELLOW((byte) 1),
diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java
index 0bb762e3ff744..19c64965e6941 100644
--- a/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java
+++ b/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -61,8 +62,9 @@
 /**
  * Cluster Index Health Information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ClusterIndexHealth implements Iterable, Writeable, ToXContentFragment {
     private static final String STATUS = "status";
     private static final String NUMBER_OF_SHARDS = "number_of_shards";
diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java
index cbc63565228f9..1fe88f65248c2 100644
--- a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java
+++ b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java
@@ -37,6 +37,7 @@
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -58,8 +59,9 @@
 /**
  * Cluster shard health information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ClusterShardHealth implements Writeable, ToXContentFragment {
     private static final String STATUS = "status";
     private static final String ACTIVE_SHARDS = "active_shards";
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java
index 31e37078d8b6c..8b3cc3c3cc2cc 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java
@@ -36,6 +36,7 @@
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.compress.CompressedXContent;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.common.xcontent.XContentFactory;
@@ -61,8 +62,9 @@
 /**
  * Metadata for index aliases
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AliasMetadata extends AbstractDiffable implements ToXContentFragment {
 
     private final String alias;
@@ -271,8 +273,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Builder of alias metadata.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
 
         private final String alias;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/CryptoMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/CryptoMetadata.java
index 27803cb106005..8bb25ba8e8472 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/CryptoMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/CryptoMetadata.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.action.admin.cluster.crypto.CryptoSettings;
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -24,8 +25,9 @@
 /**
  * Metadata about encryption and decryption
  *
- * @opensearch.internal
+ * @opensearch.experimental
  */
+@ExperimentalApi
 public class CryptoMetadata implements Writeable {
     static final public String CRYPTO_METADATA_KEY = "crypto_metadata";
     static final public String KEY_PROVIDER_NAME_KEY = "key_provider_name";
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java b/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java
index 753f872f88b22..54df245b1b835 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/DataStream.java
@@ -37,6 +37,7 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -59,8 +60,9 @@
 /**
  * Primary DataStream class
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class DataStream extends AbstractDiffable implements ToXContentObject {
 
     public static final String BACKING_INDEX_PREFIX = ".ds-";
@@ -258,8 +260,9 @@ public int hashCode() {
     /**
      * A timestamp field.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static final class TimestampField implements Writeable, ToXContentObject {
 
         static ParseField NAME_FIELD = new ParseField("name");
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
index 5d8b751b241e2..2e1421c278879 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -44,6 +44,7 @@
 import org.opensearch.cluster.node.DiscoveryNodeFilters;
 import org.opensearch.cluster.routing.allocation.IndexMetadataUpdater;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.common.compress.CompressedXContent;
 import org.opensearch.common.settings.Setting;
@@ -450,8 +451,9 @@ public Iterator> settings() {
     /**
      * Blocks the API.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum APIBlock implements Writeable {
         READ_ONLY("read_only", INDEX_READ_ONLY_BLOCK),
         READ("read", INDEX_READ_BLOCK),
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java
index c5efb55316b84..3d532208bcfe2 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java
@@ -273,8 +273,9 @@ public String toString() {
     /**
      * Builder of index template metadata.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
 
         private static final Set VALID_FIELDS = Sets.newHashSet(
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java
index 2081a0035303b..e8180613c0fa3 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java
@@ -35,6 +35,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.compress.CompressedXContent;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentHelper;
@@ -56,8 +57,9 @@
 /**
  * Mapping configuration for a type.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MappingMetadata extends AbstractDiffable {
     public static final MappingMetadata EMPTY_MAPPINGS = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, Collections.emptyMap());
 
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java
index 580643b96e411..b9d2a3edf356f 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoryMetadata.java
@@ -32,6 +32,7 @@
 package org.opensearch.cluster.metadata;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
 /**
  * Metadata about registered repository
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RepositoryMetadata implements Writeable {
 
     private final String name;
diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java
index 561f1b727efe0..2ebcd8096893d 100644
--- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java
+++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java
@@ -37,6 +37,7 @@
 import org.opensearch.cluster.Diff;
 import org.opensearch.common.Booleans;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.common.Strings;
@@ -62,8 +63,9 @@
  * This class holds all {@link DiscoveryNode} in the cluster and provides convenience methods to
  * access, modify merge / diff discovery nodes.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DiscoveryNodes extends AbstractDiffable implements Iterable {
 
     public static final DiscoveryNodes EMPTY_NODES = builder().build();
@@ -567,8 +569,9 @@ public String toString() {
     /**
      * Delta between nodes.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Delta {
 
         private final String localNodeId;
@@ -735,8 +738,9 @@ public static Builder builder(DiscoveryNodes nodes) {
     /**
      * Builder of a map of discovery nodes.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
 
         private final Map nodes;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java
index b01e074ce40c2..3dcf4ae3c9b99 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -56,8 +57,9 @@
  * - {@link SnapshotRecoverySource} recovery from a snapshot
  * - {@link LocalShardsRecoverySource} recovery from other shards of another index on the same node
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class RecoverySource implements Writeable, ToXContentObject {
 
     @Override
@@ -111,8 +113,9 @@ protected void writeAdditionalFields(StreamOutput out) throws IOException {
     /**
      * Type of recovery.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Type {
         EMPTY_STORE,
         EXISTING_STORE,
diff --git a/server/src/main/java/org/opensearch/cluster/routing/ShardRoutingState.java b/server/src/main/java/org/opensearch/cluster/routing/ShardRoutingState.java
index a46464d8727ee..2086159790ba9 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/ShardRoutingState.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/ShardRoutingState.java
@@ -32,12 +32,15 @@
 
 package org.opensearch.cluster.routing;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Represents the current state of a {@link ShardRouting} as defined by the
  * cluster.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum ShardRoutingState {
     /**
      * The shard is not assigned to any node.
diff --git a/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java b/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java
index 5e748df5eed2d..cf6dc9cd7306e 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/UnassignedInfo.java
@@ -38,6 +38,7 @@
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.cluster.routing.allocation.decider.Decision;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
@@ -62,8 +63,9 @@
 /**
  * Holds additional information as to why the shard is in unassigned state.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class UnassignedInfo implements ToXContentFragment, Writeable {
 
     public static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(ZoneOffset.UTC);
@@ -81,8 +83,9 @@ public final class UnassignedInfo implements ToXContentFragment, Writeable {
      * Note, ordering of the enum is important, make sure to add new values
      * at the end and handle version serialization properly.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Reason {
         /**
          * Unassigned as a result of an API creation of an index.
@@ -157,8 +160,9 @@ public enum Reason {
      * Note, ordering of the enum is important, make sure to add new values
      * at the end and handle version serialization properly.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum AllocationStatus implements Writeable {
         /**
          * The shard was denied allocation to a node because the allocation deciders all returned a NO decision
diff --git a/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java b/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java
index 01471ab664294..2b93a1483b801 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.cluster.routing;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -20,8 +21,9 @@
 /**
  * Entity for Weighted Round Robin weights
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class WeightedRouting implements Writeable {
     private String attributeName;
     private Map weights;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java
index 6c84957d6a788..c07fb0135262b 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RerouteExplanation.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.cluster.routing.allocation.command.AllocationCommand;
 import org.opensearch.cluster.routing.allocation.decider.Decision;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentObject;
@@ -45,8 +46,9 @@
  * Class encapsulating the explanation for a single {@link AllocationCommand}
  * taken from the Deciders
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RerouteExplanation implements ToXContentObject {
 
     private AllocationCommand command;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java
index 769212703b48b..490eb76ab8563 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingExplanations.java
@@ -33,6 +33,7 @@
 package org.opensearch.cluster.routing.allocation;
 
 import org.opensearch.cluster.routing.allocation.decider.Decision;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentFragment;
@@ -48,8 +49,9 @@
  * Class used to encapsulate a number of {@link RerouteExplanation}
  * explanations.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RoutingExplanations implements ToXContentFragment {
     private final List explanations;
 
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java
index 807ab070b82b1..ac5a18c3fcb21 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/Decision.java
@@ -33,6 +33,7 @@
 package org.opensearch.cluster.routing.allocation.decider;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -54,8 +55,9 @@
  *
  * @see AllocationDecider
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class Decision implements ToXContent, Writeable {
 
     public static final Decision ALWAYS = new Single(Type.YES);
@@ -98,8 +100,9 @@ public static Decision readFrom(StreamInput in) throws IOException {
      * This enumeration defines the
      * possible types of decisions
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Type implements Writeable {
         YES(1),
         THROTTLE(2),
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java
index afc4e36ec352e..827f3a12fbce4 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java
@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterStateTaskExecutor;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
@@ -133,7 +134,10 @@ protected ThrottlingKey registerClusterManagerTask(String taskKey, boolean throt
 
     /**
      * Class to store the throttling key for the tasks of cluster manager
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ThrottlingKey {
         private String taskThrottlingKey;
         private boolean throttlingEnabled;
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
index aa7766979e851..e097803d86b48 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
@@ -45,7 +45,6 @@
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.routing.OperationRouting;
 import org.opensearch.cluster.routing.RerouteService;
-import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
@@ -61,9 +60,8 @@
 /**
  * Main Cluster Service
  *
- * @opensearch.api
+ * @opensearch.internal
  */
-@PublicApi(since = "1.0.0")
 public class ClusterService extends AbstractLifecycleComponent {
     private final ClusterManagerService clusterManagerService;
 
diff --git a/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java b/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java
index b1723cca518fd..b06c537e7bac5 100644
--- a/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java
+++ b/server/src/main/java/org/opensearch/cluster/service/PendingClusterTask.java
@@ -33,6 +33,7 @@
 package org.opensearch.cluster.service;
 
 import org.opensearch.common.Priority;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
 /**
  * Represents a task that is pending in the cluster
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PendingClusterTask implements Writeable {
 
     private long insertOrder;
diff --git a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java
index 86a2fe1397cec..2e04f39e48c4c 100644
--- a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java
+++ b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -47,8 +48,9 @@
 /**
  * A reusable class to encode {@code field -> memory size} mappings
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class FieldMemoryStats implements Writeable, Iterable> {
 
     private final Map stats;
diff --git a/server/src/main/java/org/opensearch/common/Priority.java b/server/src/main/java/org/opensearch/common/Priority.java
index 09a751362c945..4f03d6e363550 100644
--- a/server/src/main/java/org/opensearch/common/Priority.java
+++ b/server/src/main/java/org/opensearch/common/Priority.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -40,8 +41,9 @@
 /**
  * Priority levels.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum Priority {
 
     IMMEDIATE((byte) 0),
diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java
index 4bdce11ff4f9a..1264551401b4c 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java
@@ -17,6 +17,8 @@
 
 /**
  * ReadContext is used to encapsulate all data needed by BlobContainer#readBlobAsync
+ *
+ * @opensearch.experimental
  */
 @ExperimentalApi
 public class ReadContext {
@@ -55,8 +57,11 @@ public List getPartStreams() {
     /**
      * Functional interface defining an instance that can create an async action
      * to create a part of an object represented as an InputStreamContainer.
+     *
+     * @opensearch.experimental
      */
     @FunctionalInterface
+    @ExperimentalApi
     public interface StreamPartCreator extends Supplier> {
         /**
          * Kicks off a async process to start streaming.
diff --git a/server/src/main/java/org/opensearch/common/document/DocumentField.java b/server/src/main/java/org/opensearch/common/document/DocumentField.java
index 8ce672d4fb3fc..5cdc2bba8be16 100644
--- a/server/src/main/java/org/opensearch/common/document/DocumentField.java
+++ b/server/src/main/java/org/opensearch/common/document/DocumentField.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.common.document;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -56,8 +57,9 @@
  * @see SearchHit
  * @see GetResult
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DocumentField implements Writeable, ToXContentFragment, Iterable {
 
     private final String name;
diff --git a/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java b/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java
index 0a5a66ef54c9c..d5b761a531e7f 100644
--- a/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java
+++ b/server/src/main/java/org/opensearch/common/geo/ShapeRelation.java
@@ -33,6 +33,7 @@
 package org.opensearch.common.geo;
 
 import org.apache.lucene.document.ShapeField.QueryRelation;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
  * Enum representing the relationship between a Query / Filter Shape and indexed Shapes
  * that will be used to determine if a Document should be matched or not
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum ShapeRelation implements Writeable {
 
     INTERSECTS("intersects"),
diff --git a/server/src/main/java/org/opensearch/common/inject/Module.java b/server/src/main/java/org/opensearch/common/inject/Module.java
index e66044ff26c40..b1fc031192ea0 100644
--- a/server/src/main/java/org/opensearch/common/inject/Module.java
+++ b/server/src/main/java/org/opensearch/common/inject/Module.java
@@ -29,8 +29,6 @@
 
 package org.opensearch.common.inject;
 
-import org.opensearch.common.annotation.PublicApi;
-
 /**
  * A module contributes configuration information, typically interface
  * bindings, which will be used to create an {@link Injector}. A Guice-based
@@ -45,9 +43,8 @@
  * Use scope and binding annotations on these methods to configure the
  * bindings.
  *
- * @opensearch.api
+ * @opensearch.internal
  */
-@PublicApi(since = "1.0.0")
 public interface Module {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java b/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java
index ad09efde72009..bfed1f0883672 100644
--- a/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java
+++ b/server/src/main/java/org/opensearch/common/io/stream/BytesStreamOutput.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.PageCacheRecycler;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -49,8 +50,9 @@
  * A @link {@link StreamOutput} that uses {@link BigArrays} to acquire pages of
  * bytes, which avoids frequent reallocation & copying of the internal data.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BytesStreamOutput extends BytesStream {
 
     protected final BigArrays bigArrays;
diff --git a/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java
index 98b4ebbc330da..d4dbb953ffe12 100644
--- a/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java
+++ b/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java
@@ -35,6 +35,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A logger that logs deprecation notices. Logger should be initialized with a parent logger which name will be used
@@ -50,8 +51,9 @@
  * key is combined with the X-Opaque-Id request header value, if supplied, which allows for per-client
  * message limiting.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DeprecationLogger {
 
     /**
@@ -108,8 +110,9 @@ public DeprecationLoggerBuilder deprecate(final String key, final String msg, fi
     /**
      * The builder for the deprecation logger
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public class DeprecationLoggerBuilder {
 
         public DeprecationLoggerBuilder withDeprecation(String key, String msg, Object[] params) {
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index 62e8faf33e1fa..b34a2aaffe408 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -38,6 +38,7 @@
 import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.util.FeatureFlags;
@@ -72,8 +73,9 @@
  * Encapsulates all valid index level settings.
  * @see Property#IndexScope
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class IndexScopedSettings extends AbstractScopedSettings {
 
     public static final Predicate INDEX_SETTINGS_KEY_PREDICATE = (s) -> s.startsWith(IndexMetadata.INDEX_SETTING_PREFIX);
diff --git a/server/src/main/java/org/opensearch/common/unit/Fuzziness.java b/server/src/main/java/org/opensearch/common/unit/Fuzziness.java
index 2ce86cbfd13f3..d16ec9723fb2b 100644
--- a/server/src/main/java/org/opensearch/common/unit/Fuzziness.java
+++ b/server/src/main/java/org/opensearch/common/unit/Fuzziness.java
@@ -32,6 +32,7 @@
 package org.opensearch.common.unit;
 
 import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -49,8 +50,9 @@
  * parsing and conversion from similarities to edit distances
  * etc.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class Fuzziness implements ToXContentFragment, Writeable {
 
     public static final String X_FIELD_NAME = "fuzziness";
diff --git a/server/src/main/java/org/opensearch/common/unit/SizeValue.java b/server/src/main/java/org/opensearch/common/unit/SizeValue.java
index 766199ebbc8f8..14f2bedde53f8 100644
--- a/server/src/main/java/org/opensearch/common/unit/SizeValue.java
+++ b/server/src/main/java/org/opensearch/common/unit/SizeValue.java
@@ -33,6 +33,7 @@
 package org.opensearch.common.unit;
 
 import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -43,8 +44,9 @@
 /**
  * Conversion values.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SizeValue implements Writeable, Comparable {
 
     private final long size;
diff --git a/server/src/main/java/org/opensearch/http/HttpChannel.java b/server/src/main/java/org/opensearch/http/HttpChannel.java
index 679a5d73c7837..ed20ec89a9099 100644
--- a/server/src/main/java/org/opensearch/http/HttpChannel.java
+++ b/server/src/main/java/org/opensearch/http/HttpChannel.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.http;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.network.CloseableChannel;
 import org.opensearch.core.action.ActionListener;
 
@@ -41,8 +42,9 @@
 /**
  * Represents an HTTP comms channel
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface HttpChannel extends CloseableChannel {
     /**
      * Notify HTTP channel that exception happens and the response may not be sent (for example, timeout)
diff --git a/server/src/main/java/org/opensearch/http/HttpRequest.java b/server/src/main/java/org/opensearch/http/HttpRequest.java
index 162ef11a3d865..3dc10777b657a 100644
--- a/server/src/main/java/org/opensearch/http/HttpRequest.java
+++ b/server/src/main/java/org/opensearch/http/HttpRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.http;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.rest.RestRequest;
@@ -45,15 +46,17 @@
  * A basic http request abstraction. Http modules needs to implement this interface to integrate with the
  * server package's rest handling.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface HttpRequest {
 
     /**
      * Which HTTP version being used
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     enum HttpVersion {
         HTTP_1_0,
         HTTP_1_1
diff --git a/server/src/main/java/org/opensearch/http/HttpResponse.java b/server/src/main/java/org/opensearch/http/HttpResponse.java
index 3c8269f4400e7..b25df41698c79 100644
--- a/server/src/main/java/org/opensearch/http/HttpResponse.java
+++ b/server/src/main/java/org/opensearch/http/HttpResponse.java
@@ -32,12 +32,15 @@
 
 package org.opensearch.http;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * A basic http response abstraction. Http modules must implement this interface as the server package rest
  * handling needs to set http headers for a response.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface HttpResponse {
 
     void addHeader(String name, String value);
diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
index e29283724ebf8..545623287bae8 100644
--- a/server/src/main/java/org/opensearch/index/IndexModule.java
+++ b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -496,8 +496,9 @@ IndexEventListener freeze() { // pkg private for testing
     /**
      * Type of file system
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Type {
         HYBRIDFS("hybridfs"),
         NIOFS("niofs"),
diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
index 9edb268a5126c..af2e22c4aad53 100644
--- a/server/src/main/java/org/opensearch/index/IndexSortConfig.java
+++ b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.search.SortedSetSortField;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.fielddata.IndexFieldData;
@@ -69,8 +70,10 @@
  *     
  * 
  *
- * @opensearch.internal
-**/
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "1.0.0")
 public final class IndexSortConfig {
     /**
      * The list of field names
diff --git a/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java b/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java
index b2b7781a20d26..a93a362a70c78 100644
--- a/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java
+++ b/server/src/main/java/org/opensearch/index/MergeSchedulerConfig.java
@@ -33,6 +33,7 @@
 package org.opensearch.index;
 
 import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
@@ -64,8 +65,9 @@
  *     move higher to allow merges to keep up with ongoing indexing.
  * 
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class MergeSchedulerConfig {
 
     public static final Setting MAX_THREAD_COUNT_SETTING = new Setting<>(
diff --git a/server/src/main/java/org/opensearch/index/ReplicationStats.java b/server/src/main/java/org/opensearch/index/ReplicationStats.java
index 0ae4526365bf1..8987a492e9a90 100644
--- a/server/src/main/java/org/opensearch/index/ReplicationStats.java
+++ b/server/src/main/java/org/opensearch/index/ReplicationStats.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.index;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -20,8 +21,9 @@
  * ReplicationStats is used to provide segment replication statistics at an index,
  * node and cluster level on a segment replication enabled cluster.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.10.0")
 public class ReplicationStats implements ToXContentFragment, Writeable {
 
     public long maxBytesBehind;
diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java
index c3b4f8217c961..884686ee48fa1 100644
--- a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.index;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -21,8 +22,9 @@
 /**
  * Return Segment Replication stats for a Replication Group.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class SegmentReplicationPerGroupStats implements Writeable, ToXContentFragment {
 
     private final ShardId shardId;
diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java
index 2be0c712f64ef..e381ade253422 100644
--- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java
@@ -9,6 +9,7 @@
 package org.opensearch.index;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -23,8 +24,9 @@
 /**
  * SegRep stats for a single shard.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class SegmentReplicationShardStats implements Writeable, ToXContentFragment {
     private final String allocationId;
     private final long checkpointsBehindCount;
diff --git a/server/src/main/java/org/opensearch/index/VersionType.java b/server/src/main/java/org/opensearch/index/VersionType.java
index 8aa1fbd1b81ac..01cf73ca950d0 100644
--- a/server/src/main/java/org/opensearch/index/VersionType.java
+++ b/server/src/main/java/org/opensearch/index/VersionType.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.index;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.uid.Versions;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -42,8 +43,9 @@
 /**
  * Types of index versions
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum VersionType implements Writeable {
     INTERNAL((byte) 0) {
         @Override
diff --git a/server/src/main/java/org/opensearch/index/analysis/AnalysisMode.java b/server/src/main/java/org/opensearch/index/analysis/AnalysisMode.java
index 4385680d9eb93..af71b470711a0 100644
--- a/server/src/main/java/org/opensearch/index/analysis/AnalysisMode.java
+++ b/server/src/main/java/org/opensearch/index/analysis/AnalysisMode.java
@@ -32,13 +32,16 @@
 
 package org.opensearch.index.analysis;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Enum representing the mode in which token filters and analyzers are allowed to operate.
  * While most token filters are allowed both in index and search time analyzers, some are
  * restricted to be used only at index time, others at search time.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum AnalysisMode {
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/analysis/AnalyzerScope.java b/server/src/main/java/org/opensearch/index/analysis/AnalyzerScope.java
index 0ee51b32aab46..b24d932123b44 100644
--- a/server/src/main/java/org/opensearch/index/analysis/AnalyzerScope.java
+++ b/server/src/main/java/org/opensearch/index/analysis/AnalyzerScope.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.index.analysis;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Enum to identify the scope of an analyzer
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum AnalyzerScope {
     INDEX,
     INDICES,
diff --git a/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java b/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java
index 8bc5c2ce916be..b425f17a85d6a 100644
--- a/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java
+++ b/server/src/main/java/org/opensearch/index/analysis/NameOrDefinition.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.analysis;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -48,8 +49,9 @@
 /**
  * Provides the name and settings for an analyzer
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NameOrDefinition implements Writeable, ToXContentFragment {
     // exactly one of these two members is not null
     public final String name;
diff --git a/server/src/main/java/org/opensearch/index/analysis/NamedAnalyzer.java b/server/src/main/java/org/opensearch/index/analysis/NamedAnalyzer.java
index 2a88d375a7df8..07523f9ee6dc5 100644
--- a/server/src/main/java/org/opensearch/index/analysis/NamedAnalyzer.java
+++ b/server/src/main/java/org/opensearch/index/analysis/NamedAnalyzer.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.index.mapper.MapperException;
 
 import java.util.ArrayList;
@@ -44,8 +45,9 @@
  * Named analyzer is an analyzer wrapper around an actual analyzer ({@link #analyzer} that is associated
  * with a name ({@link #name()}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NamedAnalyzer extends DelegatingAnalyzerWrapper {
 
     private final String name;
diff --git a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java
index 88077aafb1495..d844e5cbb8897 100644
--- a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.cache.query;
 
 import org.apache.lucene.search.DocIdSet;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -46,8 +47,9 @@
 /**
  * Stats for the query cache
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class QueryCacheStats implements Writeable, ToXContentFragment {
 
     private long ramBytesUsed;
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 24f68899c2ac7..6def55fcb985b 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.cache.request;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * Request for the query cache statistics
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RequestCacheStats implements Writeable, ToXContentFragment {
 
     private long memorySize;
diff --git a/server/src/main/java/org/opensearch/index/engine/CommitStats.java b/server/src/main/java/org/opensearch/index/engine/CommitStats.java
index 4328ebed85aa1..b30ce720b2649 100644
--- a/server/src/main/java/org/opensearch/index/engine/CommitStats.java
+++ b/server/src/main/java/org/opensearch/index/engine/CommitStats.java
@@ -32,6 +32,7 @@
 package org.opensearch.index.engine;
 
 import org.apache.lucene.index.SegmentInfos;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -47,8 +48,9 @@
 /**
  * a class the returns dynamic information with respect to the last commit point of this shard
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class CommitStats implements Writeable, ToXContentFragment {
 
     private final Map userData;
diff --git a/server/src/main/java/org/opensearch/index/engine/Segment.java b/server/src/main/java/org/opensearch/index/engine/Segment.java
index 41e6d79e8a4be..7881abcf58e0c 100644
--- a/server/src/main/java/org/opensearch/index/engine/Segment.java
+++ b/server/src/main/java/org/opensearch/index/engine/Segment.java
@@ -40,6 +40,7 @@
 import org.apache.lucene.search.SortedSetSortField;
 import org.opensearch.Version;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -53,8 +54,9 @@
 /**
  * A segment in the engine
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Segment implements Writeable {
 
     private String name;
diff --git a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java
index f4fd2490c7abe..d4a97f0267222 100644
--- a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java
+++ b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.engine;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -50,8 +51,9 @@
 /**
  * Tracker for segment stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SegmentsStats implements Writeable, ToXContentFragment {
 
     private long count;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java
index fc7db7d316214..85b435e969bfa 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.common.FieldMemoryStats;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -47,8 +48,9 @@
 /**
  * Encapsulates heap usage for field data
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FieldDataStats implements Writeable, ToXContentFragment {
 
     private static final String FIELDDATA = "fielddata";
diff --git a/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java
index 5860cfe8a47c3..eb3462743593d 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/ScriptDocValues.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.opensearch.common.Numbers;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.geo.GeoPoint;
 import org.opensearch.common.geo.GeoUtils;
 import org.opensearch.common.time.DateUtils;
@@ -60,8 +61,9 @@
  * return as a single {@link ScriptDocValues} instance can be reused to return
  * values form multiple documents.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ScriptDocValues extends AbstractList {
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/fielddata/SortedBinaryDocValues.java b/server/src/main/java/org/opensearch/index/fielddata/SortedBinaryDocValues.java
index bd66d1d94f33c..ba1b890f1ad1a 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/SortedBinaryDocValues.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/SortedBinaryDocValues.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.fielddata;
 
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.io.IOException;
 
@@ -41,8 +42,9 @@
  * according to {@link BytesRef#compareTo(BytesRef)}.
  * There might be dups however.
  *
- * @opensearch.internal
+  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 // TODO: Should it expose a count (current approach) or return null when there are no more values?
 public abstract class SortedBinaryDocValues {
 
diff --git a/server/src/main/java/org/opensearch/index/flush/FlushStats.java b/server/src/main/java/org/opensearch/index/flush/FlushStats.java
index aec6e9199729d..9bce46d1dd9d5 100644
--- a/server/src/main/java/org/opensearch/index/flush/FlushStats.java
+++ b/server/src/main/java/org/opensearch/index/flush/FlushStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.flush;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
 /**
  * Encapsulates statistics for flush
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FlushStats implements Writeable, ToXContentFragment {
 
     private long total;
diff --git a/server/src/main/java/org/opensearch/index/get/GetResult.java b/server/src/main/java/org/opensearch/index/get/GetResult.java
index f0f1a86cb16e7..c0dd1cd2ecb30 100644
--- a/server/src/main/java/org/opensearch/index/get/GetResult.java
+++ b/server/src/main/java/org/opensearch/index/get/GetResult.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.document.DocumentField;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.common.Strings;
@@ -68,6 +69,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetResult implements Writeable, Iterable, ToXContentObject {
 
     public static final String _INDEX = "_index";
diff --git a/server/src/main/java/org/opensearch/index/get/GetStats.java b/server/src/main/java/org/opensearch/index/get/GetStats.java
index 5530a45717305..a366014fe228e 100644
--- a/server/src/main/java/org/opensearch/index/get/GetStats.java
+++ b/server/src/main/java/org/opensearch/index/get/GetStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.get;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,6 +47,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class GetStats implements Writeable, ToXContentFragment {
 
     private long existsCount;
diff --git a/server/src/main/java/org/opensearch/index/merge/MergeStats.java b/server/src/main/java/org/opensearch/index/merge/MergeStats.java
index a284cec247ff1..7ecaed60735b4 100644
--- a/server/src/main/java/org/opensearch/index/merge/MergeStats.java
+++ b/server/src/main/java/org/opensearch/index/merge/MergeStats.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.merge;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
 /**
  * Stores stats about a merge process
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MergeStats implements Writeable, ToXContentFragment {
 
     private long total;
diff --git a/server/src/main/java/org/opensearch/index/query/IntervalMode.java b/server/src/main/java/org/opensearch/index/query/IntervalMode.java
index 454d867b41da2..a2979135ac03f 100644
--- a/server/src/main/java/org/opensearch/index/query/IntervalMode.java
+++ b/server/src/main/java/org/opensearch/index/query/IntervalMode.java
@@ -9,6 +9,7 @@
 package org.opensearch.index.query;
 
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -18,8 +19,9 @@
 /**
  * Mode for Text and Mapped Field Types
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum IntervalMode implements Writeable {
     ORDERED(0),
     UNORDERED(1),
diff --git a/server/src/main/java/org/opensearch/index/query/QueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryBuilder.java
index 090f74c5be7fe..0cdf7f31c2ebf 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryBuilder.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.query;
 
 import org.apache.lucene.search.Query;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.NamedWriteable;
 import org.opensearch.core.xcontent.ToXContentObject;
 
@@ -41,8 +42,9 @@
 /**
  * Foundation class for all OpenSearch query builders
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface QueryBuilder extends NamedWriteable, ToXContentObject, Rewriteable {
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java b/server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java
index af5a125f9dd95..b40dcca17e45b 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryBuilderVisitor.java
@@ -9,10 +9,14 @@
 package org.opensearch.index.query;
 
 import org.apache.lucene.search.BooleanClause;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * QueryBuilderVisitor is an interface to define Visitor Object to be traversed in QueryBuilder tree.
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.11.0")
 public interface QueryBuilderVisitor {
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java
index 1f7805ec99cd4..15a6d0b5a774e 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java
@@ -32,6 +32,7 @@
 package org.opensearch.index.query;
 
 import org.opensearch.client.Client;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.concurrent.CountDown;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -46,8 +47,9 @@
 /**
  * Context object used to rewrite {@link QueryBuilder} instances into simplified version.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class QueryRewriteContext {
     private final NamedXContentRegistry xContentRegistry;
     private final NamedWriteableRegistry writeableRegistry;
diff --git a/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java b/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java
index 675f7c08c827c..48fe75eb54280 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RecoveryStats.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.index.recovery;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
  * Recovery related statistics, starting at the shard level and allowing aggregation to
  * indices and node level
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RecoveryStats implements ToXContentFragment, Writeable {
 
     private final AtomicInteger currentAsSource = new AtomicInteger();
diff --git a/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java b/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java
index 322271ace82b5..489ac386f72a0 100644
--- a/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java
+++ b/server/src/main/java/org/opensearch/index/refresh/RefreshStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.refresh;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -45,8 +46,9 @@
 /**
  * Encapsulates stats for index refresh
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RefreshStats implements Writeable, ToXContentFragment {
 
     private long total;
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
index 4228ec60c4524..fc1f245829b84 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -26,8 +27,9 @@
  * Tracks remote store segment download and upload stats
  * Used for displaying remote store stats in IndicesStats/NodeStats API
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.10.0")
 public class RemoteSegmentStats implements Writeable, ToXContentFragment {
     /**
      * Cumulative bytes attempted to be uploaded to remote store
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
index fb65d9ef83be2..fe9440813b94f 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.common.CheckedFunction;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.util.Streak;
 import org.opensearch.common.util.concurrent.ConcurrentCollections;
@@ -36,8 +37,9 @@
 /**
  * Keeps track of remote refresh which happens in {@link org.opensearch.index.shard.RemoteStoreRefreshListener}. This consist of multiple critical metrics.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.10.0")
 public class RemoteSegmentTransferTracker extends RemoteTransferTracker {
 
     private final Logger logger;
@@ -391,8 +393,9 @@ public RemoteSegmentTransferTracker.Stats stats() {
     /**
      * Represents the tracker's state as seen in the stats API.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "2.10.0")
     public static class Stats implements Writeable {
 
         public final ShardId shardId;
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java
index 4214a87049350..25bee10e1fbe3 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteTranslogTransferTracker.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.index.remote;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.MovingAverage;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -22,8 +23,9 @@
 /**
  * Stores Remote Translog Store-related stats for a given IndexShard.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.10.0")
 public class RemoteTranslogTransferTracker extends RemoteTransferTracker {
     /**
      * Epoch timestamp of the last successful Remote Translog Store upload.
@@ -292,8 +294,9 @@ public String toString() {
     /**
      * Represents the tracker's state as seen in the stats API.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "2.10.0")
     public static class Stats implements Writeable {
 
         final ShardId shardId;
diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java
index 1f9144b28f286..b13f1eb04a941 100644
--- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java
+++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.search.SearchPhaseName;
 import org.opensearch.action.search.SearchRequestStats;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.Strings;
@@ -55,15 +56,17 @@
 /**
  * Encapsulates stats for search time
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchStats implements Writeable, ToXContentFragment {
 
     /**
      * Holds statistic values for a particular phase.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class PhaseStatsLongHolder implements Writeable {
 
         long current;
@@ -110,8 +113,9 @@ public void writeTo(StreamOutput out) throws IOException {
     /**
      * Holds requests stats for different phases.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class RequestStatsLongHolder {
 
         Map requestStatsHolder = new HashMap<>();
@@ -130,9 +134,9 @@ public Map getRequestStatsHolder() {
     /**
      * Holder of statistics values
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
-
+    @PublicApi(since = "1.0.0")
     public static class Stats implements Writeable, ToXContentFragment {
 
         private long queryCount;
diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java
index 216b654f6ce6a..255ff115555d8 100644
--- a/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java
+++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLease.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.seqno;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -51,8 +52,9 @@
  * otherwise merge away operations that have been soft deleted). Each retention lease contains a unique identifier, the retaining sequence
  * number, the timestamp of when the lease was created or renewed, and the source of the retention lease (e.g., "ccr").
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class RetentionLease implements ToXContentObject, Writeable {
 
     private final String id;
diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java
index 512ef9bf4f0a5..d34d385c66eb6 100644
--- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java
+++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeaseStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.seqno;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -45,8 +46,9 @@
 /**
  * Represents retention lease stats.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class RetentionLeaseStats implements ToXContentFragment, Writeable {
 
     private final RetentionLeases retentionLeases;
diff --git a/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java b/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java
index 74c4ddb6ebdb5..4e28e19c4b164 100644
--- a/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java
+++ b/server/src/main/java/org/opensearch/index/seqno/RetentionLeases.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.seqno;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -57,8 +58,9 @@
  * Represents a versioned collection of retention leases. We version the collection of retention leases to ensure that sync requests that
  * arrive out of order on the replica, using the version to ensure that older sync requests are rejected.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RetentionLeases implements ToXContentFragment, Writeable {
 
     private final long primaryTerm;
diff --git a/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java b/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java
index be7888ada2801..a8acf1fac7846 100644
--- a/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java
+++ b/server/src/main/java/org/opensearch/index/seqno/SeqNoStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.seqno;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * Sequence number statistics
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SeqNoStats implements ToXContentFragment, Writeable {
 
     private static final String SEQ_NO = "seq_no";
diff --git a/server/src/main/java/org/opensearch/index/shard/DocsStats.java b/server/src/main/java/org/opensearch/index/shard/DocsStats.java
index 83cc69752db2f..4ca475a45c04b 100644
--- a/server/src/main/java/org/opensearch/index/shard/DocsStats.java
+++ b/server/src/main/java/org/opensearch/index/shard/DocsStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.shard;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * Document statistics
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DocsStats implements Writeable, ToXContentFragment {
 
     private long count = 0;
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java
index 89cbc59403faf..862962dc5467a 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexingStats.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexingStats.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.shard;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -50,22 +51,25 @@
 /**
  * Tracks indexing statistics
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexingStats implements Writeable, ToXContentFragment {
 
     /**
      * Internal statistics for indexing
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Stats implements Writeable, ToXContentFragment {
 
         /**
          * Tracks item level rest category class codes during indexing
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public static class DocStatusStats implements Writeable, ToXContentFragment {
 
             final AtomicLong[] docStatusCounter;
diff --git a/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java
index 7ad48cb56a33b..2ab615677dedb 100644
--- a/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java
+++ b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java
@@ -9,6 +9,7 @@
 package org.opensearch.index.store;
 
 import org.apache.lucene.store.Directory;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.MovingAverage;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -22,8 +23,9 @@
 /**
  * Tracks the amount of bytes transferred between two {@link Directory} instances
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.10.0")
 public class DirectoryFileTransferTracker {
     /**
      * Cumulative size of files (in bytes) attempted to be transferred over from the source {@link Directory}
@@ -166,8 +168,9 @@ public DirectoryFileTransferTracker.Stats stats() {
     /**
      * Represents the tracker's stats presentable to an API.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "2.10.0")
     public static class Stats implements Writeable {
         public final long transferredBytesStarted;
         public final long transferredBytesFailed;
diff --git a/server/src/main/java/org/opensearch/index/store/StoreStats.java b/server/src/main/java/org/opensearch/index/store/StoreStats.java
index aa73a2c629515..4763b5e5e8a21 100644
--- a/server/src/main/java/org/opensearch/index/store/StoreStats.java
+++ b/server/src/main/java/org/opensearch/index/store/StoreStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.store;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * Statistics about an OpenSearch Store
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class StoreStats implements Writeable, ToXContentFragment {
 
     /**
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java
index ebb6202c86ec9..070fd663896a3 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.index.store.remote.filecache;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -20,8 +21,9 @@
 /**
  * Statistics on file cache
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class FileCacheStats implements Writeable, ToXContentFragment {
 
     private final long timestamp;
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteTranslogStats.java b/server/src/main/java/org/opensearch/index/translog/RemoteTranslogStats.java
index 966f8ebc2875a..03c15e0ea4752 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteTranslogStats.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteTranslogStats.java
@@ -9,6 +9,7 @@
 package org.opensearch.index.translog;
 
 import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -23,8 +24,9 @@
 /**
  * Encapsulates the stats related to Remote Translog Store operations
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.10.0")
 public class RemoteTranslogStats implements ToXContentFragment, Writeable {
     /**
      * Total number of Remote Translog Store uploads that have been started
diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java
index b44aa6e059224..9f80bc38a8222 100644
--- a/server/src/main/java/org/opensearch/index/translog/Translog.java
+++ b/server/src/main/java/org/opensearch/index/translog/Translog.java
@@ -37,6 +37,7 @@
 import org.opensearch.Version;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
@@ -1566,8 +1567,9 @@ public String toString() {
     /**
      * How to sync the translog
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Durability {
 
         /**
diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java
index a4699cea671a0..619dd6371c553 100644
--- a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java
+++ b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java
@@ -32,6 +32,7 @@
 package org.opensearch.index.translog;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
 /**
  * Translog statistics
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TranslogStats implements Writeable, ToXContentFragment {
     private static final String TRANSLOG = "translog";
     private long translogSizeInBytes;
diff --git a/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java b/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java
index 375dcc029fa70..789baaea20d04 100644
--- a/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java
+++ b/server/src/main/java/org/opensearch/index/warmer/WarmerStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.index.warmer;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
 /**
  * Stats collected about the warmer
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class WarmerStats implements Writeable, ToXContentFragment {
 
     private long current;
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
index 0f3025369833d..d91bfc19ee833 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.RateLimiter;
 import org.apache.lucene.store.RateLimiter.SimpleRateLimiter;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
@@ -48,8 +49,9 @@
 /**
  * Settings for the recovery mechanism
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RecoverySettings {
 
     private static final Logger logger = LogManager.getLogger(RecoverySettings.class);
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java
index 6733a29c19db3..80c05399df627 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryState.java
@@ -36,6 +36,7 @@
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -54,15 +55,17 @@
 /**
  * Keeps track of state related to shard recovery.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RecoveryState implements ReplicationState, ToXContentFragment, Writeable {
 
     /**
      * The stage of the recovery state
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Stage {
         INIT((byte) 0),
 
@@ -370,8 +373,9 @@ static final class Fields {
     /**
      * Verifys the lucene index
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class VerifyIndex extends ReplicationTimer implements ToXContentFragment, Writeable {
         private volatile long checkIndexTime;
 
@@ -412,8 +416,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * The translog
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Translog extends ReplicationTimer implements ToXContentFragment, Writeable {
         public static final int UNKNOWN = -1;
 
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java
index 9b5b1f4a1468e..5fa123948c5ac 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -29,15 +30,17 @@
 /**
  * ReplicationState implementation to track Segment Replication events.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.2.0")
 public class SegmentReplicationState implements ReplicationState, ToXContentFragment, Writeable {
 
     /**
      * The stage of the recovery state
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "2.2.0")
     public enum Stage {
         DONE((byte) 0),
         INIT((byte) 1),
diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java
index 5793356678715..cc52d2bafc614 100644
--- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java
+++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationLuceneIndex.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.indices.replication.common;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -32,8 +33,9 @@
  * Represents the Lucene Index (set of files on a single shard) involved
  * in the replication process.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ReplicationLuceneIndex extends ReplicationTimer implements ToXContentFragment, Writeable {
     private final FilesDetails filesDetails;
 
@@ -393,8 +395,9 @@ public boolean isComplete() {
     /**
      * Metadata about a file
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static final class FileMetadata implements ToXContentObject, Writeable {
         private String name;
         private long length;
diff --git a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java
index aeb9ef29f4a52..d884e1676f2be 100644
--- a/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java
+++ b/server/src/main/java/org/opensearch/indices/replication/common/ReplicationTimer.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.indices.replication.common;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -19,8 +20,9 @@
  * A serializable timer that is used to measure the time taken for
  * file replication operations like recovery.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ReplicationTimer implements Writeable {
     private long startTime = 0;
     private long startNanoTime = 0;
diff --git a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java
index d59d9d6da5717..477be3e74f1c7 100644
--- a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java
+++ b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.ParseField;
@@ -56,8 +57,9 @@
 /**
  * Encapsulates a pipeline's id and configuration as a blob
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class PipelineConfiguration extends AbstractDiffable implements ToXContentObject {
 
     private static final ObjectParser PARSER = new ObjectParser<>("pipeline_config", true, Builder::new);
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
index 8446ab0dd6166..fc1bb86f2ad3e 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -51,15 +52,17 @@
 /**
  * FileSystem information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FsInfo implements Iterable, Writeable, ToXContentFragment {
 
     /**
      * Path for the file system
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Path implements Writeable, ToXContentObject {
 
         String path;
@@ -220,8 +223,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * The device status.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class DeviceStats implements Writeable, ToXContentFragment {
 
         final int majorDeviceNumber;
@@ -463,8 +467,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * The I/O statistics.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class IoStats implements Writeable, ToXContentFragment {
 
         private static final String OPERATIONS = "operations";
diff --git a/server/src/main/java/org/opensearch/monitor/os/OsStats.java b/server/src/main/java/org/opensearch/monitor/os/OsStats.java
index 697b86c6ba55c..cdcec733cb086 100644
--- a/server/src/main/java/org/opensearch/monitor/os/OsStats.java
+++ b/server/src/main/java/org/opensearch/monitor/os/OsStats.java
@@ -34,6 +34,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -48,8 +49,9 @@
 /**
  * Holds stats for the Operating System
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OsStats implements Writeable, ToXContentFragment {
 
     private final long timestamp;
@@ -143,8 +145,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * CPU Information.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Cpu implements Writeable, ToXContentFragment {
 
         private final short percent;
@@ -208,8 +211,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Swap information.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Swap implements Writeable, ToXContentFragment {
 
         private static final Logger logger = LogManager.getLogger(Swap.class);
@@ -276,8 +280,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * OS Memory information.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Mem implements Writeable, ToXContentFragment {
 
         private static final Logger logger = LogManager.getLogger(Mem.class);
@@ -353,8 +358,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Encapsulates basic cgroup statistics.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Cgroup implements Writeable, ToXContentFragment {
 
         private final String cpuAcctControlGroup;
@@ -546,8 +552,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         /**
          * Encapsulates CPU time statistics.
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public static class CpuStat implements Writeable, ToXContentFragment {
 
             private final long numberOfElapsedPeriods;
diff --git a/server/src/main/java/org/opensearch/plugins/CryptoKeyProviderPlugin.java b/server/src/main/java/org/opensearch/plugins/CryptoKeyProviderPlugin.java
index 47dc75c7c908a..d9b5f0d79ec8c 100644
--- a/server/src/main/java/org/opensearch/plugins/CryptoKeyProviderPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/CryptoKeyProviderPlugin.java
@@ -14,7 +14,8 @@
 
 /**
  * Crypto plugin to provide support for custom key providers.
- * @opensearch.api
+ *
+ * @opensearch.experimental
  */
 @ExperimentalApi
 public interface CryptoKeyProviderPlugin {
diff --git a/server/src/main/java/org/opensearch/plugins/PluginInfo.java b/server/src/main/java/org/opensearch/plugins/PluginInfo.java
index 67c0542943e95..dc8fd6e604d72 100644
--- a/server/src/main/java/org/opensearch/plugins/PluginInfo.java
+++ b/server/src/main/java/org/opensearch/plugins/PluginInfo.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.bootstrap.JarHell;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -59,6 +60,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PluginInfo implements Writeable, ToXContentObject {
 
     public static final String OPENSEARCH_PLUGIN_PROPERTIES = "plugin-descriptor.properties";
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesStats.java b/server/src/main/java/org/opensearch/repositories/RepositoriesStats.java
index b24e0dddd852a..cfcbc6bb88e66 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoriesStats.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoriesStats.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.repositories;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -19,8 +20,11 @@
 import java.util.List;
 
 /**
- * Encapsulates stats for multiple repositories*
+ * Encapsulates stats for multiple repositories
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.11.0")
 public class RepositoriesStats implements Writeable, ToXContentObject {
 
     List repositoryStatsSnapshots;
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java b/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java
index f09aae195f0e5..eeddf4bd76659 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoryCleanupResult.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.repositories;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.blobstore.DeleteResult;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
@@ -47,8 +48,9 @@
 /**
  * Result of a repository cleanup action
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class RepositoryCleanupResult implements Writeable, ToXContentObject {
 
     public static final ObjectParser PARSER = new ObjectParser<>(
diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java
index f64774686c89d..2c397f7fc6e8e 100644
--- a/server/src/main/java/org/opensearch/rest/RestRequest.java
+++ b/server/src/main/java/org/opensearch/rest/RestRequest.java
@@ -37,6 +37,7 @@
 import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.SetOnce;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
@@ -72,6 +73,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RestRequest implements ToXContent.Params {
 
     // tchar pattern as defined by RFC7230 section 3.2.6
@@ -231,8 +233,9 @@ public static RestRequest requestWithoutParameters(
     /**
      * The method used.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Method {
         GET,
         POST,
diff --git a/server/src/main/java/org/opensearch/script/ScriptCacheStats.java b/server/src/main/java/org/opensearch/script/ScriptCacheStats.java
index be7a1c9542a60..d06ffa70235f8 100644
--- a/server/src/main/java/org/opensearch/script/ScriptCacheStats.java
+++ b/server/src/main/java/org/opensearch/script/ScriptCacheStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.script;
 
+import org.opensearch.common.annotation.DeprecatedApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -52,6 +53,7 @@
  *
  * @deprecated This class is deprecated in favor of ScriptStats and ScriptContextStats.  It is removed in OpenSearch 2.0.
  */
+@DeprecatedApi(since = "2.0.0")
 @Deprecated
 public class ScriptCacheStats implements Writeable, ToXContentFragment {
     private final Map context;
diff --git a/server/src/main/java/org/opensearch/script/ScriptContextInfo.java b/server/src/main/java/org/opensearch/script/ScriptContextInfo.java
index d3a64c207e0ce..ee4c0aa09388e 100644
--- a/server/src/main/java/org/opensearch/script/ScriptContextInfo.java
+++ b/server/src/main/java/org/opensearch/script/ScriptContextInfo.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.script;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -60,8 +61,9 @@
 /**
  * Information about a script context
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ScriptContextInfo implements ToXContentObject, Writeable {
     public final String name;
     public final ScriptMethodInfo execute;
@@ -202,8 +204,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /**
      * Script method information
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ScriptMethodInfo implements ToXContentObject, Writeable {
         public final String name, returnType;
         public final List parameters;
diff --git a/server/src/main/java/org/opensearch/script/ScriptContextStats.java b/server/src/main/java/org/opensearch/script/ScriptContextStats.java
index 16f4a7e313326..15a5596609586 100644
--- a/server/src/main/java/org/opensearch/script/ScriptContextStats.java
+++ b/server/src/main/java/org/opensearch/script/ScriptContextStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.script;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * Stats for a script context
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ScriptContextStats implements Writeable, ToXContentFragment, Comparable {
     private final String context;
     private final long compilations;
diff --git a/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java b/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java
index 30756ff702e8f..1a6626db64811 100644
--- a/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java
+++ b/server/src/main/java/org/opensearch/script/ScriptLanguagesInfo.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.script;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -91,8 +92,9 @@
  * }
  * 
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ScriptLanguagesInfo implements ToXContentObject, Writeable {
     private static final ParseField TYPES_ALLOWED = new ParseField("types_allowed");
     private static final ParseField LANGUAGE_CONTEXTS = new ParseField("language_contexts");
diff --git a/server/src/main/java/org/opensearch/script/ScriptStats.java b/server/src/main/java/org/opensearch/script/ScriptStats.java
index aeb3645242799..850f7f9d07070 100644
--- a/server/src/main/java/org/opensearch/script/ScriptStats.java
+++ b/server/src/main/java/org/opensearch/script/ScriptStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.script;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -48,8 +49,9 @@
 /**
  * Stats for scripts
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ScriptStats implements Writeable, ToXContentFragment {
     private final List contextStats;
     private final long compilations;
diff --git a/server/src/main/java/org/opensearch/script/StoredScriptSource.java b/server/src/main/java/org/opensearch/script/StoredScriptSource.java
index d1dae67d0e55f..fc19022657f9e 100644
--- a/server/src/main/java/org/opensearch/script/StoredScriptSource.java
+++ b/server/src/main/java/org/opensearch/script/StoredScriptSource.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.Diff;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.core.ParseField;
@@ -65,8 +66,9 @@
  * {@link StoredScriptSource} represents user-defined parameters for a script
  * saved in the {@link ClusterState}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class StoredScriptSource extends AbstractDiffable implements Writeable, ToXContentObject {
 
     /**
diff --git a/server/src/main/java/org/opensearch/search/Scroll.java b/server/src/main/java/org/opensearch/search/Scroll.java
index 8003316a57e5e..3e634a378717f 100644
--- a/server/src/main/java/org/opensearch/search/Scroll.java
+++ b/server/src/main/java/org/opensearch/search/Scroll.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.search;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
  * A scroll enables scrolling of search request. It holds a {@link #keepAlive()} time that
  * will control how long to keep the scrolling resources open.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class Scroll implements Writeable {
 
     private final TimeValue keepAlive;
diff --git a/server/src/main/java/org/opensearch/search/SearchExtBuilder.java b/server/src/main/java/org/opensearch/search/SearchExtBuilder.java
index 1df58666f6fdb..557269a1e45b1 100644
--- a/server/src/main/java/org/opensearch/search/SearchExtBuilder.java
+++ b/server/src/main/java/org/opensearch/search/SearchExtBuilder.java
@@ -33,6 +33,7 @@
 package org.opensearch.search;
 
 import org.opensearch.common.CheckedFunction;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.NamedWriteable;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -56,8 +57,9 @@
  *
  * @see SearchExtSpec
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class SearchExtBuilder implements NamedWriteable, ToXContentFragment {
 
     public abstract int hashCode();
diff --git a/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java b/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java
index c50f3c75213dd..20bdb71fd1923 100644
--- a/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java
+++ b/server/src/main/java/org/opensearch/search/builder/PointInTimeBuilder.java
@@ -33,6 +33,7 @@
 package org.opensearch.search.builder;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -50,8 +51,9 @@
  * A search request with a point in time will execute using the reader contexts associated with that point time
  * instead of the latest reader contexts.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public final class PointInTimeBuilder implements Writeable, ToXContentObject {
     private static final ParseField ID_FIELD = new ParseField("id");
     private static final ParseField KEEP_ALIVE_FIELD = new ParseField("keep_alive");
diff --git a/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java b/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java
index 73802cbd7a849..a9ff55b3a90b1 100644
--- a/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java
+++ b/server/src/main/java/org/opensearch/search/collapse/CollapseBuilder.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.search.collapse;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.ParsingException;
 import org.opensearch.core.common.Strings;
@@ -57,8 +58,9 @@
 /**
  * A builder that enables field collapsing on search request.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CollapseBuilder implements Writeable, ToXContentObject {
     public static final ParseField FIELD_FIELD = new ParseField("field");
     public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
diff --git a/server/src/main/java/org/opensearch/search/collapse/CollapseContext.java b/server/src/main/java/org/opensearch/search/collapse/CollapseContext.java
index 09a612a15d762..79b90d92f4daa 100644
--- a/server/src/main/java/org/opensearch/search/collapse/CollapseContext.java
+++ b/server/src/main/java/org/opensearch/search/collapse/CollapseContext.java
@@ -33,6 +33,7 @@
 
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.grouping.CollapsingTopDocsCollector;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.index.mapper.KeywordFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.mapper.NumberFieldMapper;
@@ -43,8 +44,9 @@
 /**
  * Context used for field collapsing
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CollapseContext {
     private final String fieldName;
     private final MappedFieldType fieldType;
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java
index f0ebfd6ab510a..337576890e663 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/FetchSourceContext.java
@@ -33,6 +33,7 @@
 package org.opensearch.search.fetch.subphase;
 
 import org.opensearch.common.Booleans;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.support.XContentMapValues;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.ParsingException;
@@ -55,8 +56,9 @@
 /**
  * Context used to fetch the {@code _source}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class FetchSourceContext implements Writeable, ToXContentObject {
 
     public static final ParseField INCLUDES_FIELD = new ParseField("includes", "include");
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java
index cc0723ed7a432..0e7c3cf30ccec 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.vectorhighlight.SimpleBoundaryScanner;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -68,8 +69,9 @@
  *
  * @see org.opensearch.search.builder.SearchSourceBuilder#highlight()
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class HighlightBuilder extends AbstractHighlighterBuilder {
     /** default for whether to highlight fields based on the source even if stored separately */
     public static final boolean DEFAULT_FORCE_SOURCE = false;
@@ -476,8 +478,9 @@ public HighlightBuilder rewrite(QueryRewriteContext ctx) throws IOException {
     /**
      * Field for highlight builder
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Field extends AbstractHighlighterBuilder {
         static final NamedObjectParser PARSER;
         static {
@@ -584,8 +587,9 @@ public Field rewrite(QueryRewriteContext ctx) throws IOException {
     /**
      * Order for highlight builder
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Order implements Writeable {
         NONE,
         SCORE;
@@ -615,8 +619,9 @@ public String toString() {
     /**
      * Boundary scanner type
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum BoundaryScannerType implements Writeable {
         CHARS,
         WORD,
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/SearchHighlightContext.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/SearchHighlightContext.java
index 7464ba094b97e..c3d34dbdef56c 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/SearchHighlightContext.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/SearchHighlightContext.java
@@ -33,6 +33,7 @@
 package org.opensearch.search.fetch.subphase.highlight;
 
 import org.apache.lucene.search.Query;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder.BoundaryScannerType;
 
 import java.util.Arrays;
@@ -46,8 +47,9 @@
 /**
  * Search context used during highlighting phase
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchHighlightContext {
 
     private final Map fields;
@@ -82,8 +84,9 @@ public boolean forceSource(Field field) {
     /**
      * Field for the search highlight context
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Field {
         private final String field;
         private final FieldOptions fieldOptions;
@@ -107,8 +110,9 @@ public FieldOptions fieldOptions() {
     /**
      * Field options for the search highlight context
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class FieldOptions {
 
         // Field options that default to null or -1 are often set to their real default in HighlighterParseElement#parse
diff --git a/server/src/main/java/org/opensearch/search/internal/AliasFilter.java b/server/src/main/java/org/opensearch/search/internal/AliasFilter.java
index c1d425078049c..1732c0ab0db8a 100644
--- a/server/src/main/java/org/opensearch/search/internal/AliasFilter.java
+++ b/server/src/main/java/org/opensearch/search/internal/AliasFilter.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.search.internal;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -47,8 +48,9 @@
 /**
  * Represents a {@link QueryBuilder} and a list of alias names that filters the builder is composed of.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class AliasFilter implements Writeable, Rewriteable {
 
     private final String[] aliases;
diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
index bc82814e88bbd..2ed770be60458 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
@@ -11,6 +11,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.ParseField;
@@ -33,7 +34,10 @@
  * TODO: Copied verbatim from {@link org.opensearch.ingest.PipelineConfiguration}.
  * 
                              
  * See if we can refactor into a common class. I suspect not, just because this one will hold
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class PipelineConfiguration extends AbstractDiffable implements ToXContentObject {
     private static final ObjectParser PARSER = new ObjectParser<>(
         "pipeline_config",
diff --git a/server/src/main/java/org/opensearch/search/profile/NetworkTime.java b/server/src/main/java/org/opensearch/search/profile/NetworkTime.java
index 45d8c2883cb4a..89a8836d807ae 100644
--- a/server/src/main/java/org/opensearch/search/profile/NetworkTime.java
+++ b/server/src/main/java/org/opensearch/search/profile/NetworkTime.java
@@ -9,6 +9,7 @@
 package org.opensearch.search.profile;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -18,8 +19,9 @@
 /**
  * Utility class to track time of network operations
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NetworkTime implements Writeable {
     private long inboundNetworkTime;
     private long outboundNetworkTime;
diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java
index 6019bc0479c21..2c0d2cf3ba78a 100644
--- a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java
+++ b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java
@@ -33,6 +33,7 @@
 package org.opensearch.search.profile;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -64,8 +65,9 @@
  * Each InternalProfileResult has a List of InternalProfileResults, which will contain
  * "children" queries if applicable
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ProfileResult implements Writeable, ToXContentObject {
     static final ParseField TYPE = new ParseField("type");
     static final ParseField DESCRIPTION = new ParseField("description");
diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java
index 502d8e4852588..8ff622152ee70 100644
--- a/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java
+++ b/server/src/main/java/org/opensearch/search/profile/ProfileShardResult.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.search.profile;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -46,8 +47,9 @@
 /**
  * Shard level profile results
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ProfileShardResult implements Writeable {
 
     private final List queryProfileResults;
diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java
index 046e929821ab5..6cb92b3efaac3 100644
--- a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java
+++ b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResult.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.search.profile.aggregation;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -51,8 +52,9 @@
  * A container class to hold the profile results for a single shard in the request.
  * Contains a list of query profiles, a collector tree and a total rewrite tree.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class AggregationProfileShardResult implements Writeable, ToXContentFragment {
 
     public static final String AGGREGATIONS = "aggregations";
diff --git a/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java b/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java
index 59d35ddd1a8d0..5cb6942445638 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java
@@ -33,6 +33,7 @@
 package org.opensearch.search.profile.query;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -55,8 +56,9 @@
  * Collectors used in the search.  Children CollectorResult's may be
  * embedded inside of a parent CollectorResult
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CollectorResult implements ToXContentObject, Writeable {
 
     public static final String REASON_SEARCH_COUNT = "search_count";
diff --git a/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java b/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java
index a9f3d4aaf7885..76d6229159698 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/QueryProfileShardResult.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.search.profile.query;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -51,8 +52,9 @@
  * A container class to hold the profile results for a single shard in the request.
  * Contains a list of query profiles, a collector tree and a total rewrite tree.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class QueryProfileShardResult implements Writeable, ToXContentObject {
 
     public static final String COLLECTOR = "collector";
diff --git a/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java b/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java
index 8df44e3958083..c9b8a896ed525 100644
--- a/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java
+++ b/server/src/main/java/org/opensearch/search/slice/SliceBuilder.java
@@ -40,7 +40,7 @@
 import org.opensearch.cluster.routing.GroupShardsIterator;
 import org.opensearch.cluster.routing.ShardIterator;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.logging.DeprecationLogger;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
@@ -76,12 +76,10 @@
  *  Otherwise the provided field must be a numeric and doc_values must be enabled. In that case a
  *  {@link org.opensearch.search.slice.DocValuesSliceQuery} is used to filter the results.
  *
- *  @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SliceBuilder implements Writeable, ToXContentObject {
-
-    private static final DeprecationLogger DEPRECATION_LOG = DeprecationLogger.getLogger(SliceBuilder.class);
-
     public static final ParseField FIELD_FIELD = new ParseField("field");
     public static final ParseField ID_FIELD = new ParseField("id");
     public static final ParseField MAX_FIELD = new ParseField("max");
diff --git a/server/src/main/java/org/opensearch/search/sort/SortOrder.java b/server/src/main/java/org/opensearch/search/sort/SortOrder.java
index ed83a0667c5e7..f4b6701c91f58 100644
--- a/server/src/main/java/org/opensearch/search/sort/SortOrder.java
+++ b/server/src/main/java/org/opensearch/search/sort/SortOrder.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.search.sort;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -43,8 +44,9 @@
 /**
  * A sorting order.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum SortOrder implements Writeable {
     /**
      * Ascending order.
diff --git a/server/src/main/java/org/opensearch/search/suggest/Suggest.java b/server/src/main/java/org/opensearch/search/suggest/Suggest.java
index abd73b45ecc13..b8f2f9b7279cf 100644
--- a/server/src/main/java/org/opensearch/search/suggest/Suggest.java
+++ b/server/src/main/java/org/opensearch/search/suggest/Suggest.java
@@ -34,6 +34,7 @@
 import org.apache.lucene.util.CollectionUtil;
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.SetOnce;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.ParsingException;
 import org.opensearch.core.common.Strings;
@@ -70,8 +71,9 @@
 /**
  * Top level suggest result, containing the result for each suggestion.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Suggest implements Iterable>>, Writeable, ToXContentFragment {
 
     public static final String NAME = "suggest";
@@ -232,7 +234,9 @@ public int hashCode() {
 
     /**
      * The suggestion responses corresponding with the suggestions in the request.
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public abstract static class Suggestion implements Iterable, NamedWriteable, ToXContentFragment {
 
         public static final int TYPE = 0;
@@ -423,7 +427,10 @@ protected static > void parseEntries(
 
         /**
          * Represents a part from the suggest text with suggested options.
+         *
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public abstract static class Entry implements Iterable, Writeable, ToXContentFragment {
 
             private static final String TEXT = "text";
@@ -611,7 +618,10 @@ protected static void declareCommonFields(ObjectParser {
 
     protected abstract Suggest.Suggestion> innerExecute(
diff --git a/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java
index 05574bbe06fa6..ee465fc527ea3 100644
--- a/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java
+++ b/server/src/main/java/org/opensearch/search/suggest/SuggestionBuilder.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.BytesRefs;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.ParsingException;
@@ -54,8 +55,9 @@
 /**
  * Base class for the different suggestion implementations.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class SuggestionBuilder> implements NamedWriteable, ToXContentFragment {
 
     protected final String field;
diff --git a/server/src/main/java/org/opensearch/search/suggest/SuggestionSearchContext.java b/server/src/main/java/org/opensearch/search/suggest/SuggestionSearchContext.java
index f0d8efc64b6b3..32f72d1115973 100644
--- a/server/src/main/java/org/opensearch/search/suggest/SuggestionSearchContext.java
+++ b/server/src/main/java/org/opensearch/search/suggest/SuggestionSearchContext.java
@@ -33,6 +33,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.index.query.QueryShardContext;
 
 import java.util.LinkedHashMap;
@@ -41,8 +42,9 @@
 /**
  * Context used for suggestion based search
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SuggestionSearchContext {
 
     private final Map suggestions = new LinkedHashMap<>(4);
@@ -58,8 +60,9 @@ public Map suggestions() {
     /**
      * The suggestion context
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public abstract static class SuggestionContext {
 
         private BytesRef text;
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java
index b54f988218ac8..ad91e75c591f3 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.common.FieldMemoryStats;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -45,8 +46,9 @@
 /**
  * Stats for completion suggester
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CompletionStats implements Writeable, ToXContentFragment {
 
     private static final String COMPLETION = "completion";
diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java b/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java
index 5a628eb7f04a0..2ccbf308705bc 100644
--- a/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java
+++ b/server/src/main/java/org/opensearch/snapshots/RestoreInfo.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.snapshots;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -53,8 +54,9 @@
  * 
                              
  * Returned as part of {@link org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse}
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RestoreInfo implements ToXContentObject, Writeable {
 
     private String name;
diff --git a/server/src/main/java/org/opensearch/snapshots/Snapshot.java b/server/src/main/java/org/opensearch/snapshots/Snapshot.java
index e7c92195eff08..4dd930c7b59c0 100644
--- a/server/src/main/java/org/opensearch/snapshots/Snapshot.java
+++ b/server/src/main/java/org/opensearch/snapshots/Snapshot.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.snapshots;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -42,8 +43,9 @@
 /**
  * Basic information about a snapshot - a SnapshotId and the repository that the snapshot belongs to.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class Snapshot implements Writeable {
 
     private final String repository;
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java
index aec3aebd93585..4eeb956a0cb19 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.snapshots;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * SnapshotId - snapshot name + snapshot UUID
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class SnapshotId implements Comparable, Writeable, ToXContentObject {
 
     private static final String NAME = "name";
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java
index 41452671e6a91..191b872cdd563 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java
@@ -35,6 +35,7 @@
 import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.opensearch.cluster.SnapshotsInProgress;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.ParseField;
@@ -64,8 +65,9 @@
 /**
  * Information about a snapshot
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class SnapshotInfo implements Comparable, ToXContent, Writeable {
 
     public static final String CONTEXT_MODE_PARAM = "context_mode";
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java
index 9d47a4edbfa22..5efcd5f12e37b 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardFailure.java
@@ -35,6 +35,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.action.ShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -52,8 +53,9 @@
 /**
  * Stores information about failures that occurred during shard snapshotting process
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotShardFailure extends ShardOperationFailedException {
 
     @Nullable
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotState.java b/server/src/main/java/org/opensearch/snapshots/SnapshotState.java
index dd1b3ebb8404d..7ad838f741a3f 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotState.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotState.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.snapshots;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Represents the state that a snapshot can be in
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum SnapshotState {
     /**
      * Snapshot process has started
diff --git a/server/src/main/java/org/opensearch/tasks/Task.java b/server/src/main/java/org/opensearch/tasks/Task.java
index ee2eae60eae8f..a21a454a65d0e 100644
--- a/server/src/main/java/org/opensearch/tasks/Task.java
+++ b/server/src/main/java/org/opensearch/tasks/Task.java
@@ -32,9 +32,8 @@
 
 package org.opensearch.tasks;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.opensearch.ExceptionsHelper;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.action.NotifyOnceListener;
 import org.opensearch.core.common.io.stream.NamedWriteable;
@@ -62,12 +61,10 @@
 /**
  * Current task information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Task {
-
-    private static final Logger logger = LogManager.getLogger(Task.class);
-
     /**
      * The request header to mark tasks with specific ids
      */
@@ -501,7 +498,10 @@ public boolean supportsResourceTracking() {
      * can change this on version upgrade but we should be careful
      * because some statuses (reindex) have become defacto standardized because
      * they are used by systems like Kibana.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public interface Status extends ToXContentObject, NamedWriteable {}
 
     /**
diff --git a/server/src/main/java/org/opensearch/tasks/TaskInfo.java b/server/src/main/java/org/opensearch/tasks/TaskInfo.java
index b51240f234e39..3a04e8e4072b2 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskInfo.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskInfo.java
@@ -33,6 +33,7 @@
 package org.opensearch.tasks;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
@@ -66,8 +67,9 @@
  * and use in APIs. Instead, immutable and writeable TaskInfo objects are used to represent
  * snapshot information about currently running tasks.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class TaskInfo implements Writeable, ToXContentFragment {
     private final TaskId taskId;
 
diff --git a/server/src/main/java/org/opensearch/tasks/TaskResult.java b/server/src/main/java/org/opensearch/tasks/TaskResult.java
index f42e833f5ca08..846fbde48ea59 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskResult.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskResult.java
@@ -34,6 +34,7 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.client.Requests;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -63,8 +64,9 @@
  * Information about a running task or a task that stored its result. Running tasks just have a {@link #getTask()} while
  * tasks with stored result will have either a {@link #getError()} or {@link #getResponse()}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class TaskResult implements Writeable, ToXContentObject {
     private final boolean completed;
     private final TaskInfo task;
diff --git a/server/src/main/java/org/opensearch/threadpool/ExecutorBuilder.java b/server/src/main/java/org/opensearch/threadpool/ExecutorBuilder.java
index d26131484ba71..d57b74b3a45e0 100644
--- a/server/src/main/java/org/opensearch/threadpool/ExecutorBuilder.java
+++ b/server/src/main/java/org/opensearch/threadpool/ExecutorBuilder.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.threadpool;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
@@ -44,8 +45,9 @@
  *
  * @param  the underlying type of the executor settings
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ExecutorBuilder {
 
     private final String name;
diff --git a/server/src/main/java/org/opensearch/threadpool/Scheduler.java b/server/src/main/java/org/opensearch/threadpool/Scheduler.java
index 4a7c63a1b559a..9733db29f5939 100644
--- a/server/src/main/java/org/opensearch/threadpool/Scheduler.java
+++ b/server/src/main/java/org/opensearch/threadpool/Scheduler.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.ExceptionsHelper;
 import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.AbstractRunnable;
@@ -154,7 +155,10 @@ static ScheduledCancellable wrapAsScheduledCancellable(ScheduledFuture schedu
 
     /**
      * This interface represents an object whose execution may be cancelled during runtime.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     interface Cancellable {
 
         /**
@@ -171,7 +175,10 @@ interface Cancellable {
 
     /**
      * A scheduled cancellable allow cancelling and reading the remaining delay of a scheduled task.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     interface ScheduledCancellable extends Delayed, Cancellable {}
 
     /**
diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
index 5f10986239300..c825ecc8abe9f 100644
--- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java
@@ -122,8 +122,9 @@ public static class Names {
     /**
      * The threadpool type.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum ThreadPoolType {
         DIRECT("direct"),
         FIXED("fixed"),
@@ -734,8 +735,9 @@ ExecutorService executor() {
     /**
      * The thread pool information.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Info implements Writeable, ToXContentFragment {
 
         private final String name;
diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java b/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java
index f49ad34e70119..49d961f0b506f 100644
--- a/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java
+++ b/server/src/main/java/org/opensearch/threadpool/ThreadPoolInfo.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.threadpool;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.service.ReportingService;
@@ -45,8 +46,9 @@
 /**
  * Information about a threadpool
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ThreadPoolInfo implements ReportingService.Info, Iterable {
 
     private final List infos;
diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java
index 7b4c1504d927a..968c2cc4c4887 100644
--- a/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java
+++ b/server/src/main/java/org/opensearch/threadpool/ThreadPoolStats.java
@@ -33,6 +33,7 @@
 package org.opensearch.threadpool;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -49,15 +50,17 @@
 /**
  * Stats for a threadpool
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ThreadPoolStats implements Writeable, ToXContentFragment, Iterable {
 
     /**
      * The statistics.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Stats implements Writeable, ToXContentFragment, Comparable {
 
         private final String name;
diff --git a/server/src/main/java/org/opensearch/watcher/ResourceWatcher.java b/server/src/main/java/org/opensearch/watcher/ResourceWatcher.java
index 0c656c52388df..a54ba60815883 100644
--- a/server/src/main/java/org/opensearch/watcher/ResourceWatcher.java
+++ b/server/src/main/java/org/opensearch/watcher/ResourceWatcher.java
@@ -31,6 +31,8 @@
 
 package org.opensearch.watcher;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.IOException;
 
 /**
@@ -39,8 +41,9 @@
  * Different resource watchers can be registered with {@link ResourceWatcherService} to be called
  * periodically in order to check for changes in different external resources.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ResourceWatcher {
     /**
      * Called once when the resource watcher is added to {@link ResourceWatcherService}
diff --git a/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java b/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java
index 62002f9c6c323..519d7c6e68a57 100644
--- a/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java
+++ b/server/src/main/java/org/opensearch/watcher/ResourceWatcherService.java
@@ -64,8 +64,9 @@ public class ResourceWatcherService implements Closeable {
     /**
      * Frequency level to watch.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum Frequency {
 
         /**
diff --git a/server/src/main/java/org/opensearch/watcher/WatcherHandle.java b/server/src/main/java/org/opensearch/watcher/WatcherHandle.java
index 6890364cd0cd6..dd8f898e11860 100644
--- a/server/src/main/java/org/opensearch/watcher/WatcherHandle.java
+++ b/server/src/main/java/org/opensearch/watcher/WatcherHandle.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.watcher;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Handle to a watcher
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class WatcherHandle {
 
     private final ResourceWatcherService.ResourceMonitor monitor;

From dcb64d8facde23587da9b09e92e587b68d022e94 Mon Sep 17 00:00:00 2001
From: Andrew Ross 
Date: Fri, 10 Nov 2023 12:35:29 -0600
Subject: [PATCH 285/388] Make the gradle check comment more compact (#11151)

Continuing in the spirit of #9699, this makes the gradle check comment
less verbose while containing all the same information. The goal is to
reduce visual noise on the PR feed and maybe allow for more comments
before GitHub starts hiding older comments behind the "load more" link.

This is super subjective so happy to hear any opinions if the previous
format is preferred.

Signed-off-by: Andrew Ross 
---
 .github/workflows/gradle-check.yml | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
index 31578b3fc4660..8c33d41c6b2b4 100644
--- a/.github/workflows/gradle-check.yml
+++ b/.github/workflows/gradle-check.yml
@@ -82,10 +82,7 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :white_check_mark:
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
+            :white_check_mark: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }})
 
       - name: Extract Test Failure
         if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
@@ -108,10 +105,8 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
+            :grey_exclamation: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }}) ${{ env.test_failures }}
+
             Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
 
       - name: Create Comment Failure
@@ -120,12 +115,9 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-            ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
-            * **URL:** ${{ env.workflow_url }}
-            * **CommitID:** ${{ env.pr_from_sha }}
-            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
-            Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
+            :x: Gradle check result for ${{ env.pr_from_sha }}: [${{ env.result }}](${{ env.workflow_url }})
+
+            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green. Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
 
       - name: Create Issue On Push Failure
         if: ${{ github.event_name == 'push' && failure() }}

From 2bd709bf201dde294e21d9959d951bb7f9e5e40a Mon Sep 17 00:00:00 2001
From: Neetika Singhal 
Date: Fri, 10 Nov 2023 14:13:30 -0800
Subject: [PATCH 286/388] update index random function to fix the bogus
 documents deletion (#11142)

Signed-off-by: Neetika Singhal 
---
 .../basic/TransportTwoNodesSearchIT.java      |  1 +
 .../highlight/HighlighterSearchIT.java        | 16 ----------------
 .../test/OpenSearchIntegTestCase.java         | 19 +++++++++++++------
 3 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java
index ce5f3f63faa66..edceb0cbc0d24 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java
@@ -128,6 +128,7 @@ private Set prepareData(int numShards) throws Exception {
             fullExpectedIds.add(Integer.toString(i));
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
         return fullExpectedIds;
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index f7bc5eb75ad0f..2afa911223074 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -2013,10 +2013,6 @@ private static String randomStoreField() {
     }
 
     public void testHighlightNoMatchSize() throws IOException, InterruptedException {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         assertAcked(
             prepareCreate("test").setMapping(
                 "text",
@@ -2128,10 +2124,6 @@ public void testHighlightNoMatchSize() throws IOException, InterruptedException
     }
 
     public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException, InterruptedException {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         assertAcked(
             prepareCreate("test").setMapping(
                 "text",
@@ -2228,10 +2220,6 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException,
     }
 
     public void testHighlightNoMatchSizeNumberOfFragments() throws IOException, InterruptedException {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         assertAcked(
             prepareCreate("test").setMapping(
                 "text",
@@ -3416,10 +3404,6 @@ public void testFiltersFunctionScoreQueryHighlight() throws Exception {
     }
 
     public void testHighlightQueryRewriteDatesWithNow() throws Exception {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10434",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         assertAcked(
             client().admin()
                 .indices()
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index 0c6c81103922f..7614cd0e8f920 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -1667,10 +1667,6 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
         }
         assertThat(actualErrors, emptyIterable());
 
-        if (dummyDocuments) {
-            bogusIds.addAll(indexRandomForMultipleSlices(indicesArray));
-        }
-
         if (!bogusIds.isEmpty()) {
             // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
             for (List doc : bogusIds) {
@@ -1686,6 +1682,9 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
                 client().admin().indices().prepareRefresh(indicesArray).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get()
             );
         }
+        if (dummyDocuments) {
+            indexRandomForMultipleSlices(indicesArray);
+        }
     }
 
     /*
@@ -1694,7 +1693,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
     * multiple slices based on segment count.
     * @param indices         the indices in which bogus documents should be ingested
     * */
-    protected Set> indexRandomForMultipleSlices(String... indices) throws InterruptedException {
+    protected void indexRandomForMultipleSlices(String... indices) throws InterruptedException {
         Set> bogusIds = new HashSet<>();
         int refreshCount = randomIntBetween(2, 3);
         for (String index : indices) {
@@ -1731,7 +1730,15 @@ protected Set> indexRandomForMultipleSlices(String... indices) thro
                 refresh(index);
             }
         }
-        return bogusIds;
+        for (List doc : bogusIds) {
+            assertEquals(
+                "failed to delete a dummy doc [" + doc.get(0) + "][" + doc.get(1) + "]",
+                DocWriteResponse.Result.DELETED,
+                client().prepareDelete(doc.get(0), doc.get(1)).setRouting(doc.get(1)).get().getResult()
+            );
+        }
+        // refresh is called to make sure the bogus docs doesn't affect the search results
+        refresh();
     }
 
     private final AtomicInteger dummmyDocIdGenerator = new AtomicInteger();

From 0452d1487fd76640c5345fa92aa1ce9b1bacbc88 Mon Sep 17 00:00:00 2001
From: Niyati Aggarwal <121826855+niyatiagg@users.noreply.github.com>
Date: Sat, 11 Nov 2023 16:45:34 -0800
Subject: [PATCH 287/388] Adding support for dynamically updating
 Leader/follower checker timeouts (#10528)

* making leader check timeout dynamic

Signed-off-by: Niyati Aggarwal 

* making follower check timeout dynamic

Signed-off-by: Niyati Aggarwal 

* fixing existing unit tests

Signed-off-by: Niyati Aggarwal 

* fixing checkstyle violations

Signed-off-by: Niyati Aggarwal 

* adding tests for leader/follower check timeout

Signed-off-by: Niyati Aggarwal 

* setting maximum and minimum timeout value for leader/follower checker

Signed-off-by: Niyati Aggarwal 

* adding tests for checking boundary cases

Signed-off-by: Niyati Aggarwal 

* Fixing checkstyle violations

Signed-off-by: Niyati Aggarwal 

* changed the log file and added other suggested changes

Signed-off-by: Niyati Aggarwal 

* fixing checkstyle violations

Signed-off-by: Niyati Aggarwal 

* Addressing review comments

Signed-off-by: Niyati Aggarwal 

* addressing proposed changes

Signed-off-by: Niyati Aggarwal 

* Applying checkstyle fixes

Signed-off-by: Niyati Aggarwal 

* Fixing flakiness for existing tests

Signed-off-by: Niyati Aggarwal 

* Applying checkstyle fixes

Signed-off-by: Niyati Aggarwal 

* Fixing the timeout value limits for randomSettings

Signed-off-by: Niyati Aggarwal 

---------

Signed-off-by: Niyati Aggarwal 
---
 CHANGELOG.md                                  |   1 +
 .../cluster/coordination/Coordinator.java     |   3 +-
 .../coordination/FollowersChecker.java        |  14 ++-
 .../cluster/coordination/LeaderChecker.java   |  13 +-
 .../transport/TransportChannel.java           |   2 +-
 .../CoordinationCheckerSettingsTests.java     | 113 ++++++++++++++++++
 .../coordination/FollowersCheckerTests.java   |  29 +++--
 .../coordination/LeaderCheckerTests.java      |  12 +-
 8 files changed, 164 insertions(+), 23 deletions(-)
 create mode 100644 server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e8f27b9323306..e45eb3870b522 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -129,6 +129,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
 - Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
+- Made leader/follower check timeout setting dynamic ([#10528](https://github.com/opensearch-project/OpenSearch/pull/10528))
 - Use iterative approach to evaluate Regex.simpleMatch ([#11060](https://github.com/opensearch-project/OpenSearch/pull/11060))
 
 ### Deprecated
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
index a4ffab7fb70c9..5a07f964f94a4 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
@@ -261,9 +261,10 @@ public Coordinator(
             this::handlePublishRequest,
             this::handleApplyCommit
         );
-        this.leaderChecker = new LeaderChecker(settings, transportService, this::onLeaderFailure, nodeHealthService);
+        this.leaderChecker = new LeaderChecker(settings, clusterSettings, transportService, this::onLeaderFailure, nodeHealthService);
         this.followersChecker = new FollowersChecker(
             settings,
+            clusterSettings,
             transportService,
             this::onFollowerCheckRequest,
             this::removeNode,
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java
index f69a4f771cf21..70bb0515bb022 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java
@@ -38,6 +38,7 @@
 import org.opensearch.cluster.coordination.Coordinator.Mode;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -98,7 +99,9 @@ public class FollowersChecker {
         "cluster.fault_detection.follower_check.timeout",
         TimeValue.timeValueMillis(10000),
         TimeValue.timeValueMillis(1),
-        Setting.Property.NodeScope
+        TimeValue.timeValueMillis(60000),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
     );
 
     // the number of failed checks that must happen before the follower is considered to have failed.
@@ -112,7 +115,7 @@ public class FollowersChecker {
     private final Settings settings;
 
     private final TimeValue followerCheckInterval;
-    private final TimeValue followerCheckTimeout;
+    private TimeValue followerCheckTimeout;
     private final int followerCheckRetryCount;
     private final BiConsumer onNodeFailure;
     private final Consumer handleRequestAndUpdateState;
@@ -127,6 +130,7 @@ public class FollowersChecker {
 
     public FollowersChecker(
         Settings settings,
+        ClusterSettings clusterSettings,
         TransportService transportService,
         Consumer handleRequestAndUpdateState,
         BiConsumer onNodeFailure,
@@ -141,7 +145,7 @@ public FollowersChecker(
         followerCheckInterval = FOLLOWER_CHECK_INTERVAL_SETTING.get(settings);
         followerCheckTimeout = FOLLOWER_CHECK_TIMEOUT_SETTING.get(settings);
         followerCheckRetryCount = FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings);
-
+        clusterSettings.addSettingsUpdateConsumer(FOLLOWER_CHECK_TIMEOUT_SETTING, this::setFollowerCheckTimeout);
         updateFastResponseState(0, Mode.CANDIDATE);
         transportService.registerRequestHandler(
             FOLLOWER_CHECK_ACTION_NAME,
@@ -159,6 +163,10 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
         });
     }
 
+    private void setFollowerCheckTimeout(TimeValue followerCheckTimeout) {
+        this.followerCheckTimeout = followerCheckTimeout;
+    }
+
     /**
      * Update the set of known nodes, starting to check any new ones and stopping checking any previously-known-but-now-unknown ones.
      */
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java
index 69ba1f977f326..8d4373b865f62 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java
@@ -40,6 +40,7 @@
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.lease.Releasable;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -97,7 +98,9 @@ public class LeaderChecker {
         "cluster.fault_detection.leader_check.timeout",
         TimeValue.timeValueMillis(10000),
         TimeValue.timeValueMillis(1),
-        Setting.Property.NodeScope
+        TimeValue.timeValueMillis(60000),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
     );
 
     // the number of failed checks that must happen before the leader is considered to have failed.
@@ -111,7 +114,7 @@ public class LeaderChecker {
     private final Settings settings;
 
     private final TimeValue leaderCheckInterval;
-    private final TimeValue leaderCheckTimeout;
+    private TimeValue leaderCheckTimeout;
     private final int leaderCheckRetryCount;
     private final TransportService transportService;
     private final Consumer onLeaderFailure;
@@ -123,6 +126,7 @@ public class LeaderChecker {
 
     LeaderChecker(
         final Settings settings,
+        final ClusterSettings clusterSettings,
         final TransportService transportService,
         final Consumer onLeaderFailure,
         NodeHealthService nodeHealthService
@@ -134,6 +138,7 @@ public class LeaderChecker {
         this.transportService = transportService;
         this.onLeaderFailure = onLeaderFailure;
         this.nodeHealthService = nodeHealthService;
+        clusterSettings.addSettingsUpdateConsumer(LEADER_CHECK_TIMEOUT_SETTING, this::setLeaderCheckTimeout);
 
         transportService.registerRequestHandler(
             LEADER_CHECK_ACTION_NAME,
@@ -155,6 +160,10 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
         });
     }
 
+    private void setLeaderCheckTimeout(TimeValue leaderCheckTimeout) {
+        this.leaderCheckTimeout = leaderCheckTimeout;
+    }
+
     public DiscoveryNode leader() {
         CheckScheduler checkScheduler = currentChecker.get();
         return checkScheduler == null ? null : checkScheduler.leader;
diff --git a/server/src/main/java/org/opensearch/transport/TransportChannel.java b/server/src/main/java/org/opensearch/transport/TransportChannel.java
index 7423d59103302..f84ee5dc745c3 100644
--- a/server/src/main/java/org/opensearch/transport/TransportChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TransportChannel.java
@@ -82,7 +82,7 @@ static void sendErrorResponse(TransportChannel channel, String actionName, Trans
 
     /**
      * Returns the contextual property associated with this specific transport channel (the
-     * implementation of how such properties are managed depends on the the particular
+     * implementation of how such properties are managed depends on the particular
      * transport engine).
      *
      * @param name the name of the property
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java
new file mode 100644
index 0000000000000..56bd2d94dce84
--- /dev/null
+++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java
@@ -0,0 +1,113 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.coordination;
+
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+
+import static org.opensearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING;
+import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING;
+import static org.opensearch.common.unit.TimeValue.timeValueSeconds;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+public class CoordinationCheckerSettingsTests extends OpenSearchSingleNodeTestCase {
+    public void testFollowerCheckTimeoutValueUpdate() {
+        Setting setting1 = FOLLOWER_CHECK_TIMEOUT_SETTING;
+        Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "60s").build();
+        try {
+            ClusterUpdateSettingsResponse response = client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(timeSettings1)
+                .execute()
+                .actionGet();
+
+            assertAcked(response);
+            assertEquals(timeValueSeconds(60), setting1.get(response.getPersistentSettings()));
+        } finally {
+            // cleanup
+            timeSettings1 = Settings.builder().putNull(setting1.getKey()).build();
+            client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet();
+        }
+    }
+
+    public void testFollowerCheckTimeoutMaxValue() {
+        Setting setting1 = FOLLOWER_CHECK_TIMEOUT_SETTING;
+        Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "61s").build();
+
+        assertThrows(
+            "failed to parse value [61s] for setting [" + setting1.getKey() + "], must be <= [60000ms]",
+            IllegalArgumentException.class,
+            () -> {
+                client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet();
+            }
+        );
+    }
+
+    public void testFollowerCheckTimeoutMinValue() {
+        Setting setting1 = FOLLOWER_CHECK_TIMEOUT_SETTING;
+        Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "0s").build();
+
+        assertThrows(
+            "failed to parse value [0s] for setting [" + setting1.getKey() + "], must be >= [1ms]",
+            IllegalArgumentException.class,
+            () -> {
+                client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet();
+            }
+        );
+    }
+
+    public void testLeaderCheckTimeoutValueUpdate() {
+        Setting setting1 = LEADER_CHECK_TIMEOUT_SETTING;
+        Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "60s").build();
+        try {
+            ClusterUpdateSettingsResponse response = client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(timeSettings1)
+                .execute()
+                .actionGet();
+            assertAcked(response);
+            assertEquals(timeValueSeconds(60), setting1.get(response.getPersistentSettings()));
+        } finally {
+            // cleanup
+            timeSettings1 = Settings.builder().putNull(setting1.getKey()).build();
+            client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet();
+        }
+    }
+
+    public void testLeaderCheckTimeoutMaxValue() {
+        Setting setting1 = LEADER_CHECK_TIMEOUT_SETTING;
+        Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "61s").build();
+
+        assertThrows(
+            "failed to parse value [61s] for setting [" + setting1.getKey() + "], must be <= [60000ms]",
+            IllegalArgumentException.class,
+            () -> {
+                client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet();
+            }
+        );
+    }
+
+    public void testLeaderCheckTimeoutMinValue() {
+        Setting setting1 = LEADER_CHECK_TIMEOUT_SETTING;
+        Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "0s").build();
+
+        assertThrows(
+            "failed to parse value [0s] for setting [" + setting1.getKey() + "], must be >= [1ms]",
+            IllegalArgumentException.class,
+            () -> {
+                client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet();
+            }
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java
index c152a1606681e..a106706c00732 100644
--- a/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java
+++ b/server/src/test/java/org/opensearch/cluster/coordination/FollowersCheckerTests.java
@@ -39,6 +39,7 @@
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.Settings.Builder;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -96,7 +97,7 @@ public class FollowersCheckerTests extends OpenSearchTestCase {
     public void testChecksExpectedNodes() {
         final DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getName()).build();
-
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DiscoveryNodes[] discoveryNodesHolder = new DiscoveryNodes[] {
             DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).build() };
 
@@ -132,6 +133,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req
 
         final FollowersChecker followersChecker = new FollowersChecker(
             settings,
+            clusterSettings,
             transportService,
             fcr -> { assert false : fcr; },
             (node, reason) -> {
@@ -257,6 +259,7 @@ public void testFailsNodeThatDisconnects() {
         final DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNode otherNode = new DiscoveryNode("other-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getName()).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
 
         final MockTransport mockTransport = new MockTransport() {
@@ -297,6 +300,7 @@ public String toString() {
 
         final FollowersChecker followersChecker = new FollowersChecker(
             settings,
+            clusterSettings,
             transportService,
             fcr -> { assert false : fcr; },
             (node, reason) -> {
@@ -336,6 +340,7 @@ private void testBehaviourOfFailingNode(
         final DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNode otherNode = new DiscoveryNode("other-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getName()).put(testSettings).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
 
         final MockTransport mockTransport = new MockTransport() {
@@ -384,6 +389,7 @@ public String toString() {
 
         final FollowersChecker followersChecker = new FollowersChecker(
             settings,
+            clusterSettings,
             transportService,
             fcr -> { assert false : fcr; },
             (node, reason) -> {
@@ -464,6 +470,7 @@ public void testUnhealthyNodeRejectsImmediately() {
         final DiscoveryNode leader = new DiscoveryNode("leader", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNode follower = new DiscoveryNode("follower", buildNewFakeTransportAddress(), Version.CURRENT);
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), follower.getName()).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
 
         final MockTransport mockTransport = new MockTransport() {
@@ -488,7 +495,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req
         final AtomicBoolean calledCoordinator = new AtomicBoolean();
         final AtomicReference coordinatorException = new AtomicReference<>();
 
-        final FollowersChecker followersChecker = new FollowersChecker(settings, transportService, fcr -> {
+        final FollowersChecker followersChecker = new FollowersChecker(settings, clusterSettings, transportService, fcr -> {
             assertTrue(calledCoordinator.compareAndSet(false, true));
             final RuntimeException exception = coordinatorException.get();
             if (exception != null) {
@@ -536,6 +543,7 @@ public void testResponder() {
         final DiscoveryNode leader = new DiscoveryNode("leader", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNode follower = new DiscoveryNode("follower", buildNewFakeTransportAddress(), Version.CURRENT);
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), follower.getName()).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
 
         final MockTransport mockTransport = new MockTransport() {
@@ -560,7 +568,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req
         final AtomicBoolean calledCoordinator = new AtomicBoolean();
         final AtomicReference coordinatorException = new AtomicReference<>();
 
-        final FollowersChecker followersChecker = new FollowersChecker(settings, transportService, fcr -> {
+        final FollowersChecker followersChecker = new FollowersChecker(settings, clusterSettings, transportService, fcr -> {
             assertTrue(calledCoordinator.compareAndSet(false, true));
             final RuntimeException exception = coordinatorException.get();
             if (exception != null) {
@@ -700,6 +708,7 @@ public void testPreferClusterManagerNodes() {
         DiscoveryNodes discoveryNodes = discoNodesBuilder.localNodeId(nodes.get(0).getId()).build();
         CapturingTransport capturingTransport = new CapturingTransport();
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), nodes.get(0).getName()).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
         TransportService transportService = capturingTransport.createTransportService(
             Settings.EMPTY,
@@ -710,15 +719,9 @@ public void testPreferClusterManagerNodes() {
             emptySet(),
             NoopTracer.INSTANCE
         );
-        final FollowersChecker followersChecker = new FollowersChecker(
-            Settings.EMPTY,
-            transportService,
-            fcr -> { assert false : fcr; },
-            (node, reason) -> {
-                assert false : node;
-            },
-            () -> new StatusInfo(HEALTHY, "healthy-info")
-        );
+        final FollowersChecker followersChecker = new FollowersChecker(Settings.EMPTY, clusterSettings, transportService, fcr -> {
+            assert false : fcr;
+        }, (node, reason) -> { assert false : node; }, () -> new StatusInfo(HEALTHY, "healthy-info"));
         followersChecker.setCurrentNodes(discoveryNodes);
         List followerTargets = Stream.of(capturingTransport.getCapturedRequestsAndClear())
             .map(cr -> cr.node)
@@ -754,7 +757,7 @@ private static Settings randomSettings() {
             settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms");
         }
         if (randomBoolean()) {
-            settingsBuilder.put(FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 100000) + "ms");
+            settingsBuilder.put(FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 60000) + "ms");
         }
         return settingsBuilder.build();
     }
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java
index 8915f4c5c1274..fe65058333116 100644
--- a/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java
+++ b/server/src/test/java/org/opensearch/cluster/coordination/LeaderCheckerTests.java
@@ -38,6 +38,7 @@
 import org.opensearch.cluster.coordination.LeaderChecker.LeaderCheckRequest;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.transport.TransportResponse;
@@ -117,6 +118,7 @@ public void testFollowerBehaviour() {
         final AtomicBoolean allResponsesFail = new AtomicBoolean();
 
         final Settings settings = settingsBuilder.build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         logger.info("--> using {}", settings);
 
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
@@ -174,7 +176,7 @@ public String toString() {
 
         final AtomicBoolean leaderFailed = new AtomicBoolean();
 
-        final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService, e -> {
+        final LeaderChecker leaderChecker = new LeaderChecker(settings, clusterSettings, transportService, e -> {
             assertThat(e.getMessage(), matchesRegex("node \\[.*\\] failed \\[[1-9][0-9]*\\] consecutive checks"));
             assertTrue(leaderFailed.compareAndSet(false, true));
         }, () -> new StatusInfo(StatusInfo.Status.HEALTHY, "healthy-info"));
@@ -242,6 +244,7 @@ public void testFollowerFailsImmediatelyOnDisconnection() {
         final Response[] responseHolder = new Response[] { Response.SUCCESS };
 
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getId()).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
         final MockTransport mockTransport = new MockTransport() {
             @Override
@@ -290,7 +293,7 @@ public String toString() {
         transportService.acceptIncomingRequests();
 
         final AtomicBoolean leaderFailed = new AtomicBoolean();
-        final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService, e -> {
+        final LeaderChecker leaderChecker = new LeaderChecker(settings, clusterSettings, transportService, e -> {
             assertThat(e.getMessage(), anyOf(endsWith("disconnected"), endsWith("disconnected during check")));
             assertTrue(leaderFailed.compareAndSet(false, true));
         }, () -> new StatusInfo(StatusInfo.Status.HEALTHY, "healthy-info"));
@@ -357,6 +360,7 @@ public void testFollowerFailsImmediatelyOnHealthCheckFailure() {
         final Response[] responseHolder = new Response[] { Response.SUCCESS };
 
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getId()).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
         final MockTransport mockTransport = new MockTransport() {
             @Override
@@ -403,7 +407,7 @@ public String toString() {
         transportService.acceptIncomingRequests();
 
         final AtomicBoolean leaderFailed = new AtomicBoolean();
-        final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService, e -> {
+        final LeaderChecker leaderChecker = new LeaderChecker(settings, clusterSettings, transportService, e -> {
             assertThat(e.getMessage(), endsWith("failed health checks"));
             assertTrue(leaderFailed.compareAndSet(false, true));
         }, () -> new StatusInfo(StatusInfo.Status.HEALTHY, "healthy-info"));
@@ -432,6 +436,7 @@ public void testLeaderBehaviour() {
         final DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNode otherNode = new DiscoveryNode("other-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getId()).build();
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
         final CapturingTransport capturingTransport = new CapturingTransport();
         AtomicReference nodeHealthServiceStatus = new AtomicReference<>(new StatusInfo(UNHEALTHY, "unhealthy-info"));
@@ -450,6 +455,7 @@ public void testLeaderBehaviour() {
 
         final LeaderChecker leaderChecker = new LeaderChecker(
             settings,
+            clusterSettings,
             transportService,
             e -> fail("shouldn't be checking anything"),
             () -> nodeHealthServiceStatus.get()

From b01e4831e5725fc51194f074deb90168c72def56 Mon Sep 17 00:00:00 2001
From: Ashish 
Date: Mon, 13 Nov 2023 02:10:01 +0530
Subject: [PATCH 288/388] [Remote Translog] Fix translog inconsistent
 generation assertion failure in ITs (#10985)

---
 .../index/translog/RemoteFsTranslog.java           | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
index 65d16e213cad1..8fb420e8fa1da 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -380,12 +380,14 @@ public boolean syncNeeded() {
     public void close() throws IOException {
         assert Translog.calledFromOutsideOrViaTragedyClose() : shardId
             + "Translog.close method is called from inside Translog, but not via closeOnTragicEvent method";
-        if (closed.compareAndSet(false, true)) {
-            try (ReleasableLock lock = writeLock.acquire()) {
-                sync();
-            } finally {
-                logger.debug("translog closed");
-                closeFilesIfNoPendingRetentionLocks();
+        try (ReleasableLock lock = writeLock.acquire()) {
+            if (closed.compareAndSet(false, true)) {
+                try {
+                    sync();
+                } finally {
+                    logger.debug("translog closed");
+                    closeFilesIfNoPendingRetentionLocks();
+                }
             }
         }
     }

From 8077a26e709660f522ca78420d7d38a39139030f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Nov 2023 11:41:39 -0500
Subject: [PATCH 289/388] Bump stefanzweifel/git-auto-commit-action from 4 to 5
 (#11171)

* Bump stefanzweifel/git-auto-commit-action from 4 to 5

Bumps [stefanzweifel/git-auto-commit-action](https://github.com/stefanzweifel/git-auto-commit-action) from 4 to 5.
- [Release notes](https://github.com/stefanzweifel/git-auto-commit-action/releases)
- [Changelog](https://github.com/stefanzweifel/git-auto-commit-action/blob/master/CHANGELOG.md)
- [Commits](https://github.com/stefanzweifel/git-auto-commit-action/compare/v4...v5)

---
updated-dependencies:
- dependency-name: stefanzweifel/git-auto-commit-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] 

* Update changelog

Signed-off-by: dependabot[bot] 

---------

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] 
---
 .github/workflows/dependabot_pr.yml | 6 +++---
 CHANGELOG.md                        | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml
index 2a5e539b214d3..e6feb3b852ad0 100644
--- a/.github/workflows/dependabot_pr.yml
+++ b/.github/workflows/dependabot_pr.yml
@@ -27,7 +27,7 @@ jobs:
           ./gradlew updateSHAs
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Updating SHAs
           branch: ${{ github.head_ref }}
@@ -40,7 +40,7 @@ jobs:
           ./gradlew spotlessApply
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Spotless formatting
           branch: ${{ github.head_ref }}
@@ -54,7 +54,7 @@ jobs:
           version: 'Unreleased 2.x'
 
       - name: Commit the changes
-        uses: stefanzweifel/git-auto-commit-action@v4
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: "Update changelog"
           branch: ${{ github.head_ref }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e45eb3870b522..daf5568fa7a77 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -116,6 +116,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
 - Bump `org.apache.logging.log4j:log4j-core` from 2.20.0 to 2.21.1 ([#10858](https://github.com/opensearch-project/OpenSearch/pull/10858), [#11000](https://github.com/opensearch-project/OpenSearch/pull/11000))
 - Bump `aws-actions/configure-aws-credentials` from 2 to 4 ([#10504](https://github.com/opensearch-project/OpenSearch/pull/10504))
+- Bump `stefanzweifel/git-auto-commit-action` from 4 to 5 ([#11171](https://github.com/opensearch-project/OpenSearch/pull/11171))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))

From c676479f6d1a0d8da8d3b23d05a31747a5322dac Mon Sep 17 00:00:00 2001
From: Andrew Ross 
Date: Mon, 13 Nov 2023 10:53:07 -0600
Subject: [PATCH 290/388] Mute CardinalityWithRequestBreakerIT pending fix
 (#10752)

Relates #10154

Signed-off-by: Andrew Ross 
---
 .../aggregations/metrics/CardinalityWithRequestBreakerIT.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
index 2bf5230c67b43..85c36ec0ba78d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
@@ -76,6 +76,7 @@ protected Settings featureFlagSettings() {
     /**
      * Test that searches using cardinality aggregations returns all request breaker memory.
      */
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10154")
     public void testRequestBreaker() throws Exception {
         final String requestBreaker = randomIntBetween(1, 10000) + "kb";
         logger.info("--> Using request breaker setting: {}", requestBreaker);

From 6c980bc9079c383569613a005937eedb5c97cacc Mon Sep 17 00:00:00 2001
From: Jay Deng 
Date: Mon, 13 Nov 2023 15:49:58 -0800
Subject: [PATCH 291/388] Add indexRandomForConcurrentSearch method to tests:
 (#11118)

* IndexStatsIT.java
* SearchTimeoutIT.java
* NaNSortingIT.java
* NestedIT.java
* RangeIT.java

Signed-off-by: Jay Deng 
---
 .../org/opensearch/indices/stats/IndexStatsIT.java     |  4 +++-
 .../java/org/opensearch/search/SearchTimeoutIT.java    | 10 +++++-----
 .../search/aggregations/bucket/NaNSortingIT.java       |  1 +
 .../search/aggregations/bucket/NestedIT.java           |  4 ++++
 .../opensearch/search/aggregations/bucket/RangeIT.java |  2 ++
 5 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java
index 0967acb37d3e8..9c96d4861d426 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java
@@ -169,7 +169,7 @@ private Settings.Builder settingsBuilder() {
         return Settings.builder().put(indexSettings());
     }
 
-    public void testFieldDataStats() {
+    public void testFieldDataStats() throws InterruptedException {
         assertAcked(
             client().admin()
                 .indices()
@@ -182,6 +182,7 @@ public void testFieldDataStats() {
         client().prepareIndex("test").setId("1").setSource("field", "value1", "field2", "value1").execute().actionGet();
         client().prepareIndex("test").setId("2").setSource("field", "value2", "field2", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
+        indexRandomForConcurrentSearch("test");
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(
@@ -305,6 +306,7 @@ public void testClearAllCaches() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value1").execute().actionGet();
         client().prepareIndex("test").setId("2").setSource("field", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
+        indexRandomForConcurrentSearch("test");
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java
index 94816346e6c9e..52cc797ddd8da 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java
@@ -93,6 +93,7 @@ public void testSimpleTimeout() throws Exception {
             client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get();
         }
         refresh("test");
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setTimeout(new TimeValue(5, TimeUnit.MILLISECONDS))
@@ -104,12 +105,11 @@ public void testSimpleTimeout() throws Exception {
     }
 
     public void testSimpleDoesNotTimeout() throws Exception {
-        final int numDocs = 10;
+        final int numDocs = 9;
         for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get();
+            client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         }
-        refresh("test");
-
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch("test")
             .setTimeout(new TimeValue(10000, TimeUnit.SECONDS))
             .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap())))
@@ -122,7 +122,7 @@ public void testSimpleDoesNotTimeout() throws Exception {
 
     public void testPartialResultsIntolerantTimeout() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-
+        indexRandomForConcurrentSearch("test");
         OpenSearchException ex = expectThrows(
             OpenSearchException.class,
             () -> client().prepareSearch("test")
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java
index 1ef2c0e8db8c7..6289cd5e36151 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java
@@ -172,6 +172,7 @@ public void setupSuiteScopeCluster() throws Exception {
             client().prepareIndex("idx").setSource(source.endObject()).get();
         }
         refresh();
+        indexRandomForMultipleSlices("idx");
         ensureSearchable();
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java
index b3009ffcf4f45..7af2ac218800d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java
@@ -224,6 +224,7 @@ public void setupSuiteScopeCluster() throws Exception {
                 )
         );
         indexRandom(true, builders);
+        indexRandomForMultipleSlices("idx");
         ensureSearchable();
     }
 
@@ -354,6 +355,7 @@ public void testNestedAsSubAggregation() throws Exception {
     }
 
     public void testNestNestedAggs() throws Exception {
+        indexRandomForConcurrentSearch("idx_nested_nested_aggs");
         SearchResponse response = client().prepareSearch("idx_nested_nested_aggs")
             .addAggregation(
                 nested("level1", "nested1").subAggregation(
@@ -607,6 +609,7 @@ public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("idx4");
 
         SearchResponse response = client().prepareSearch("idx4")
             .addAggregation(
@@ -782,6 +785,7 @@ public void testFilterAggInsideNestedAgg() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("classes");
 
         SearchResponse response = client().prepareSearch("classes")
             .addAggregation(
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java
index 64ab6f1382ac3..5812b7796c33e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java
@@ -184,6 +184,7 @@ public void setupSuiteScopeCluster() throws Exception {
         builders.add(client().prepareIndex("new_index").setSource(Collections.emptyMap()));
 
         indexRandom(true, builders);
+        indexRandomForMultipleSlices("idx", "old_index", "new_index");
         ensureSearchable();
     }
 
@@ -917,6 +918,7 @@ public void testOverlappingRanges() throws Exception {
     }
 
     public void testEmptyAggregation() throws Exception {
+        indexRandomForConcurrentSearch("empty_bucket_idx");
         SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
             .setQuery(matchAllQuery())
             .addAggregation(

From 5143198f43f05e5d1692624adc3733ac19b90004 Mon Sep 17 00:00:00 2001
From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com>
Date: Tue, 14 Nov 2023 20:37:22 +0530
Subject: [PATCH 292/388] Enabling sort optimization back for half_float with
 custom comparators (#11024)

* Enabling sort optimizatin back for half_float with custom comparators

Signed-off-by: Chaitanya Gohel 

* Fixing tests

Signed-off-by: Chaitanya Gohel 

* Adding test for Indecx sort half_float

Signed-off-by: Chaitanya Gohel 

* Making indexFieldData provate in FloatValuesComparatorSource

Signed-off-by: Chaitanya Gohel 

* Update server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java

Co-authored-by: Andriy Redko 
Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com>

* Adding missing value instead null

Signed-off-by: Chaitanya Gohel 

* Adding more tests for desc order sort

Signed-off-by: Chaitanya Gohel 

* Update rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml

Co-authored-by: Prabhakar Sithanandam 
Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com>

* Adding tests in case missing values are competitive

Signed-off-by: Chaitanya Gohel 

* chanheing newly added test supported version 3.0.0

Signed-off-by: Chaitanya Gohel 

* Assing missing float tests

Signed-off-by: Chaitanya Gohel 

* Remove missing value change to be part of another PR

Signed-off-by: Chaitanya Gohel 

---------

Signed-off-by: Chaitanya Gohel 
Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com>
Co-authored-by: Andriy Redko 
Co-authored-by: Prabhakar Sithanandam 
---
 CHANGELOG.md                                  |   1 +
 .../test/indices.sort/10_basic.yml            |  20 +++
 .../test/search/260_sort_mixed.yml            |  85 ++++++++++++
 .../test/search/90_search_after.yml           | 127 ++++++++++++++++++
 .../opensearch/search/sort/FieldSortIT.java   |  60 +++++++++
 .../fielddata/IndexNumericFieldData.java      |   5 +-
 .../HalfFloatValuesComparatorSource.java      |  75 +++++++++++
 .../comparators/HalfFloatComparator.java      | 111 +++++++++++++++
 8 files changed, 483 insertions(+), 1 deletion(-)
 create mode 100644 server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java
 create mode 100644 server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index daf5568fa7a77..2165fcb4700fb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [AdmissionControl] Added changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting ([#9286](https://github.com/opensearch-project/OpenSearch/pull/9286))
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
 - [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
+- Add back half_float BKD based sort query optimization ([#11024](https://github.com/opensearch-project/OpenSearch/pull/11024))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml
index b9089689b0cf1..3b7ea15164e9f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml
@@ -156,3 +156,23 @@
           query: {"range": { "rank": { "from": 0 } } }
           track_total_hits: false
           size: 3
+
+---
+"Index Sort half float":
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            index.sort.field: rank
+          mappings:
+            properties:
+              rank:
+                type: half_float
+
+  # This should failed with 400 as half_float is not supported for index sort
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml
index ba2b18eb3b6d0..a04dc308b2a06 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml
@@ -20,6 +20,7 @@
             properties:
               counter:
                 type: double
+
   - do:
       bulk:
         refresh: true
@@ -119,3 +120,87 @@
   - match: { status: 400 }
   - match: { error.type: search_phase_execution_exception }
   - match: { error.caused_by.reason: "Can't do sort across indices, as a field has [unsigned_long] type in one index, and different type in another index!" }
+
+---
+"search across indices with mixed long and double and float numeric types":
+  - skip:
+      version: " - 2.10.99"
+      reason: half float was broken before 2.11
+
+  - do:
+      indices.create:
+        index: test_1
+        body:
+          mappings:
+            properties:
+              counter:
+                type: long
+
+  - do:
+      indices.create:
+        index: test_2
+        body:
+          mappings:
+            properties:
+              counter:
+                type: double
+
+  - do:
+      indices.create:
+        index: test_3
+        body:
+          mappings:
+            properties:
+              counter:
+                type: half_float
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test_1
+          - counter: 223372036854775800
+          - index:
+              _index: test_2
+          - counter: 1223372036854775800.23
+          - index:
+              _index: test_2
+          - counter: 184.4
+          - index:
+              _index: test_3
+          - counter: 187.4
+          - index:
+              _index: test_3
+          - counter: 194.4
+
+  - do:
+      search:
+        index: test_*
+        rest_total_hits_as_int: true
+        body:
+          sort: [{ counter: desc }]
+  - match: { hits.total: 5 }
+  - length: { hits.hits: 5 }
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.0._source.counter: 1223372036854775800.23 }
+  - match: { hits.hits.0.sort.0: 1223372036854775800.23 }
+  - match: { hits.hits.1._index: test_1 }
+  - match: { hits.hits.1._source.counter: 223372036854775800 }
+  - match: { hits.hits.1.sort.0: 223372036854775800  }
+  - match: { hits.hits.2._index: test_3 }
+  - match: { hits.hits.2._source.counter: 194.4 }
+
+  - do:
+      search:
+        index: test_*
+        rest_total_hits_as_int: true
+        body:
+          sort: [{ counter: asc }]
+  - match: { hits.total: 5 }
+  - length: { hits.hits: 5 }
+  - match: { hits.hits.0._index: test_2 }
+  - match: { hits.hits.0._source.counter: 184.4 }
+  - match: { hits.hits.0.sort.0: 184.4 }
+  - match: { hits.hits.1._index: test_3 }
+  - match: { hits.hits.1._source.counter: 187.4 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml
index 55e1566656faf..1563daba9de6d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml
@@ -320,3 +320,130 @@
   - length: { hits.hits: 1 }
   - match: { hits.hits.0._index: test }
   - match: { hits.hits.0._source.population: null }
+
+---
+"half float":
+  - skip:
+        version: " - 2.10.99"
+        reason: half_float was broken for 2.10 and earlier
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              population:
+                type: half_float
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body: |
+          {"index":{}}
+          {"population": 184.4}
+          {"index":{}}
+          {"population": 194.4}
+          {"index":{}}
+          {"population": 144.4}
+          {"index":{}}
+          {"population": 174.4}
+          {"index":{}}
+          {"population": 164.4}
+
+  - do:
+      search:
+        index: test
+        rest_total_hits_as_int: true
+        body:
+          size: 3
+          sort: [ { population: desc } ]
+  - match: { hits.total: 5 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._source.population: 194.4 }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1._source.population: 184.4 }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._source.population: 174.4 }
+
+  - do:
+      search:
+        index: test
+        rest_total_hits_as_int: true
+        body:
+          size: 3
+          sort: [ { population: asc } ]
+  - match: { hits.total: 5 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._source.population: 144.4 }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1._source.population: 164.4 }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._source.population: 174.4 }
+
+  # search_after with the asc sort
+  - do:
+      search:
+        index: test
+        rest_total_hits_as_int: true
+        body:
+          size: 1
+          sort: [ { population: asc } ]
+          search_after: [ 184.375 ] # this is rounded sort value in sort result
+  - match: { hits.total: 5 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._source.population: 194.4 }
+
+  # search_after with the desc sort
+  - do:
+      search:
+        index: test
+        rest_total_hits_as_int: true
+        body:
+          size: 1
+          sort: [ { population: desc } ]
+          search_after: [ 164.375 ] # this is rounded sort value in sort result
+  - match: { hits.total: 5 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._source.population: 144.4 }
+
+  # search_after with the asc sort with missing
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body: |
+          {"index":{}}
+          {"population": null}
+  - do:
+      search:
+        index: test
+        rest_total_hits_as_int: true
+        body:
+          "size": 5
+          "sort": [ { "population": { "order": "asc", "missing": "_last" } } ]
+          search_after: [ 200 ] # making it out of min/max so only missing value hit is qualified
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._source.population: null }
+
+  # search_after with the desc sort with missing
+  - do:
+      search:
+        index: test
+        rest_total_hits_as_int: true
+        body:
+          "size": 5
+          "sort": [ { "population": { "order": "desc", "missing": "_last" } } ]
+          search_after: [ 100 ] # making it out of min/max so only missing value hit is qualified
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._source.population: null }
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
index bee242b933dfd..d4980a64a3977 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
@@ -605,6 +605,9 @@ public void testSimpleSorts() throws Exception {
                     .startObject("float_value")
                     .field("type", "float")
                     .endObject()
+                    .startObject("half_float_value")
+                    .field("type", "half_float")
+                    .endObject()
                     .startObject("double_value")
                     .field("type", "double")
                     .endObject()
@@ -628,6 +631,7 @@ public void testSimpleSorts() throws Exception {
                         .field("long_value", i)
                         .field("unsigned_long_value", UNSIGNED_LONG_BASE.add(BigInteger.valueOf(10000 * i)))
                         .field("float_value", 0.1 * i)
+                        .field("half_float_value", 0.1 * i)
                         .field("double_value", 0.1 * i)
                         .endObject()
                 );
@@ -794,6 +798,28 @@ public void testSimpleSorts() throws Exception {
 
         assertThat(searchResponse.toString(), not(containsString("error")));
 
+        // HALF_FLOAT
+        size = 1 + random.nextInt(10);
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("half_float_value", SortOrder.ASC).get();
+
+        assertHitCount(searchResponse, 10L);
+        assertThat(searchResponse.getHits().getHits().length, equalTo(size));
+        for (int i = 0; i < size; i++) {
+            assertThat(searchResponse.getHits().getAt(i).getId(), equalTo(Integer.toString(i)));
+        }
+
+        assertThat(searchResponse.toString(), not(containsString("error")));
+        size = 1 + random.nextInt(10);
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("half_float_value", SortOrder.DESC).get();
+
+        assertHitCount(searchResponse, 10);
+        assertThat(searchResponse.getHits().getHits().length, equalTo(size));
+        for (int i = 0; i < size; i++) {
+            assertThat(searchResponse.getHits().getAt(i).getId(), equalTo(Integer.toString(9 - i)));
+        }
+
+        assertThat(searchResponse.toString(), not(containsString("error")));
+
         // DOUBLE
         size = 1 + random.nextInt(10);
         searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("double_value", SortOrder.ASC).get();
@@ -1330,6 +1356,9 @@ public void testSortMVField() throws Exception {
                     .startObject("float_values")
                     .field("type", "float")
                     .endObject()
+                    .startObject("half_float_values")
+                    .field("type", "float")
+                    .endObject()
                     .startObject("double_values")
                     .field("type", "double")
                     .endObject()
@@ -1351,6 +1380,7 @@ public void testSortMVField() throws Exception {
                     .array("short_values", 1, 5, 10, 8)
                     .array("byte_values", 1, 5, 10, 8)
                     .array("float_values", 1f, 5f, 10f, 8f)
+                    .array("half_float_values", 1f, 5f, 10f, 8f)
                     .array("double_values", 1d, 5d, 10d, 8d)
                     .array("string_values", "01", "05", "10", "08")
                     .endObject()
@@ -1365,6 +1395,7 @@ public void testSortMVField() throws Exception {
                     .array("short_values", 11, 15, 20, 7)
                     .array("byte_values", 11, 15, 20, 7)
                     .array("float_values", 11f, 15f, 20f, 7f)
+                    .array("half_float_values", 11f, 15f, 20f, 7f)
                     .array("double_values", 11d, 15d, 20d, 7d)
                     .array("string_values", "11", "15", "20", "07")
                     .endObject()
@@ -1379,6 +1410,7 @@ public void testSortMVField() throws Exception {
                     .array("short_values", 2, 1, 3, -4)
                     .array("byte_values", 2, 1, 3, -4)
                     .array("float_values", 2f, 1f, 3f, -4f)
+                    .array("half_float_values", 2f, 1f, 3f, -4f)
                     .array("double_values", 2d, 1d, 3d, -4d)
                     .array("string_values", "02", "01", "03", "!4")
                     .endObject()
@@ -1585,6 +1617,34 @@ public void testSortMVField() throws Exception {
         assertThat(searchResponse.getHits().getAt(2).getId(), equalTo(Integer.toString(3)));
         assertThat(((Number) searchResponse.getHits().getAt(2).getSortValues()[0]).floatValue(), equalTo(3f));
 
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("half_float_values", SortOrder.ASC).get();
+
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(3L));
+        assertThat(searchResponse.getHits().getHits().length, equalTo(3));
+
+        assertThat(searchResponse.getHits().getAt(0).getId(), equalTo(Integer.toString(3)));
+        assertThat(((Number) searchResponse.getHits().getAt(0).getSortValues()[0]).floatValue(), equalTo(-4f));
+
+        assertThat(searchResponse.getHits().getAt(1).getId(), equalTo(Integer.toString(1)));
+        assertThat(((Number) searchResponse.getHits().getAt(1).getSortValues()[0]).floatValue(), equalTo(1f));
+
+        assertThat(searchResponse.getHits().getAt(2).getId(), equalTo(Integer.toString(2)));
+        assertThat(((Number) searchResponse.getHits().getAt(2).getSortValues()[0]).floatValue(), equalTo(7f));
+
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("half_float_values", SortOrder.DESC).get();
+
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(3L));
+        assertThat(searchResponse.getHits().getHits().length, equalTo(3));
+
+        assertThat(searchResponse.getHits().getAt(0).getId(), equalTo(Integer.toString(2)));
+        assertThat(((Number) searchResponse.getHits().getAt(0).getSortValues()[0]).floatValue(), equalTo(20f));
+
+        assertThat(searchResponse.getHits().getAt(1).getId(), equalTo(Integer.toString(1)));
+        assertThat(((Number) searchResponse.getHits().getAt(1).getSortValues()[0]).floatValue(), equalTo(10f));
+
+        assertThat(searchResponse.getHits().getAt(2).getId(), equalTo(Integer.toString(3)));
+        assertThat(((Number) searchResponse.getHits().getAt(2).getSortValues()[0]).floatValue(), equalTo(3f));
+
         searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(10).addSort("double_values", SortOrder.ASC).get();
 
         assertThat(searchResponse.getHits().getTotalHits().value, equalTo(3L));
diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
index 6fc074fe0de95..b0ff944d014de 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
@@ -42,6 +42,7 @@
 import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.opensearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
 import org.opensearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
+import org.opensearch.index.fielddata.fieldcomparator.HalfFloatValuesComparatorSource;
 import org.opensearch.index.fielddata.fieldcomparator.IntValuesComparatorSource;
 import org.opensearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
 import org.opensearch.index.fielddata.fieldcomparator.UnsignedLongValuesComparatorSource;
@@ -220,6 +221,8 @@ private XFieldComparatorSource comparatorSource(
         final XFieldComparatorSource source;
         switch (targetNumericType) {
             case HALF_FLOAT:
+                source = new HalfFloatValuesComparatorSource(this, missingValue, sortMode, nested);
+                break;
             case FLOAT:
                 source = new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
                 break;
@@ -242,7 +245,7 @@ private XFieldComparatorSource comparatorSource(
                 assert !targetNumericType.isFloatingPoint();
                 source = new IntValuesComparatorSource(this, missingValue, sortMode, nested);
         }
-        if (targetNumericType != getNumericType() || getNumericType() == NumericType.HALF_FLOAT) {
+        if (targetNumericType != getNumericType()) {
             source.disableSkipping(); // disable skipping logic for cast of sort field
         }
         return source;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java
new file mode 100644
index 0000000000000..7e3936be1d8a5
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java
@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.LeafFieldComparator;
+import org.apache.lucene.util.BitSet;
+import org.opensearch.index.fielddata.FieldData;
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+import org.opensearch.index.fielddata.NumericDoubleValues;
+import org.opensearch.index.fielddata.SortedNumericDoubleValues;
+import org.opensearch.index.search.comparators.HalfFloatComparator;
+import org.opensearch.search.MultiValueMode;
+
+import java.io.IOException;
+
+/**
+ * Comparator source for half_float values.
+ *
+ * @opensearch.internal
+ */
+public class HalfFloatValuesComparatorSource extends FloatValuesComparatorSource {
+    private final IndexNumericFieldData indexFieldData;
+
+    public HalfFloatValuesComparatorSource(
+        IndexNumericFieldData indexFieldData,
+        Object missingValue,
+        MultiValueMode sortMode,
+        Nested nested
+    ) {
+        super(indexFieldData, missingValue, sortMode, nested);
+        this.indexFieldData = indexFieldData;
+    }
+
+    @Override
+    public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) {
+        assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName());
+
+        final float fMissingValue = (Float) missingObject(missingValue, reversed);
+        // NOTE: it's important to pass null as a missing value in the constructor so that
+        // the comparator doesn't check docsWithField since we replace missing values in select()
+        return new HalfFloatComparator(numHits, fieldname, null, reversed, enableSkipping && this.enableSkipping) {
+            @Override
+            public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
+                return new HalfFloatLeafComparator(context) {
+                    @Override
+                    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
+                        return HalfFloatValuesComparatorSource.this.getNumericDocValues(context, fMissingValue).getRawFloatValues();
+                    }
+                };
+            }
+        };
+    }
+
+    private NumericDoubleValues getNumericDocValues(LeafReaderContext context, float missingValue) throws IOException {
+        final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues();
+        if (nested == null) {
+            return FieldData.replaceMissing(sortMode.select(values), missingValue);
+        } else {
+            final BitSet rootDocs = nested.rootDocs(context);
+            final DocIdSetIterator innerDocs = nested.innerDocs(context);
+            final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
+            return sortMode.select(values, missingValue, rootDocs, innerDocs, context.reader().maxDoc(), maxChildren);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java b/server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java
new file mode 100644
index 0000000000000..6244fa647b042
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/search/comparators/HalfFloatComparator.java
@@ -0,0 +1,111 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.search.comparators;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
+import org.apache.lucene.search.LeafFieldComparator;
+import org.apache.lucene.search.comparators.NumericComparator;
+
+import java.io.IOException;
+
+/**
+ * The comparator for half_float numeric type.
+ * Comparator based on {@link Float#compare} for {@code numHits}. This comparator provides a
+ * skipping functionality – an iterator that can skip over non-competitive documents.
+ */
+public class HalfFloatComparator extends NumericComparator {
+    private final float[] values;
+    protected float topValue;
+    protected float bottom;
+
+    public HalfFloatComparator(int numHits, String field, Float missingValue, boolean reverse, boolean enableSkipping) {
+        super(field, missingValue != null ? missingValue : 0.0f, reverse, enableSkipping, HalfFloatPoint.BYTES);
+        values = new float[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        return Float.compare(values[slot1], values[slot2]);
+    }
+
+    @Override
+    public void setTopValue(Float value) {
+        super.setTopValue(value);
+        topValue = value;
+    }
+
+    @Override
+    public Float value(int slot) {
+        return Float.valueOf(values[slot]);
+    }
+
+    @Override
+    public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
+        return new HalfFloatLeafComparator(context);
+    }
+
+    /** Leaf comparator for {@link HalfFloatComparator} that provides skipping functionality */
+    public class HalfFloatLeafComparator extends NumericLeafComparator {
+
+        public HalfFloatLeafComparator(LeafReaderContext context) throws IOException {
+            super(context);
+        }
+
+        private float getValueForDoc(int doc) throws IOException {
+            if (docValues.advanceExact(doc)) {
+                return Float.intBitsToFloat((int) docValues.longValue());
+            } else {
+                return missingValue;
+            }
+        }
+
+        @Override
+        public void setBottom(int slot) throws IOException {
+            bottom = values[slot];
+            super.setBottom(slot);
+        }
+
+        @Override
+        public int compareBottom(int doc) throws IOException {
+            return Float.compare(bottom, getValueForDoc(doc));
+        }
+
+        @Override
+        public int compareTop(int doc) throws IOException {
+            return Float.compare(topValue, getValueForDoc(doc));
+        }
+
+        @Override
+        public void copy(int slot, int doc) throws IOException {
+            values[slot] = getValueForDoc(doc);
+            super.copy(slot, doc);
+        }
+
+        @Override
+        protected int compareMissingValueWithBottomValue() {
+            return Float.compare(missingValue, bottom);
+        }
+
+        @Override
+        protected int compareMissingValueWithTopValue() {
+            return Float.compare(missingValue, topValue);
+        }
+
+        @Override
+        protected void encodeBottom(byte[] packedValue) {
+            HalfFloatPoint.encodeDimension(bottom, packedValue, 0);
+        }
+
+        @Override
+        protected void encodeTop(byte[] packedValue) {
+            HalfFloatPoint.encodeDimension(topValue, packedValue, 0);
+        }
+    }
+}

From 3f30511ff78ce3f0c379a22fcbd665b82a7fb216 Mon Sep 17 00:00:00 2001
From: Bansi Kasundra <66969140+kasundra07@users.noreply.github.com>
Date: Tue, 14 Nov 2023 10:09:22 -0800
Subject: [PATCH 293/388] Fixing the tests for concurrent search (#11076)

* Fixing the tests for concurrent search

Signed-off-by: Bansi Kasundra 

* Remove changes for non-flaky tests

Signed-off-by: Bansi Kasundra 

---------

Signed-off-by: Bansi Kasundra 
---
 .../search/fetch/subphase/InnerHitsIT.java    |  4 +
 .../fetch/subphase/MatchedQueriesIT.java      | 20 +++--
 .../search/query/SearchQueryIT.java           | 73 ++++++++++++++-----
 3 files changed, 74 insertions(+), 23 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java
index 9b3e1337418cc..1a730c01e4890 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java
@@ -897,6 +897,7 @@ public void testNestedSource() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("index1");
 
         // the field name (comments.message) used for source filtering should be the same as when using that field for
         // other features (like in the query dsl or aggs) in order for consistency:
@@ -973,6 +974,7 @@ public void testInnerHitsWithIgnoreUnmapped() throws Exception {
         client().prepareIndex("index1").setId("1").setSource("nested_type", Collections.singletonMap("key", "value")).get();
         client().prepareIndex("index2").setId("3").setSource("key", "value").get();
         refresh();
+        indexRandomForConcurrentSearch("index1", "index2");
 
         SearchResponse response = client().prepareSearch("index1", "index2")
             .setQuery(
@@ -1002,6 +1004,7 @@ public void testUseMaxDocInsteadOfSize() throws Exception {
             .setRefreshPolicy(IMMEDIATE)
             .get();
 
+        indexRandomForConcurrentSearch("index2");
         QueryBuilder query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit(
             new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1)
         );
@@ -1019,6 +1022,7 @@ public void testTooHighResultWindow() throws Exception {
             )
             .setRefreshPolicy(IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("index2");
         SearchResponse response = client().prepareSearch("index2")
             .setQuery(
                 nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit(
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java
index 23b5d0cab0697..83cedb8c20e1d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java
@@ -91,6 +91,7 @@ public void testSimpleMatchedQueryFromFilteredQuery() throws Exception {
         client().prepareIndex("test").setId("2").setSource("name", "test2", "number", 2).get();
         client().prepareIndex("test").setId("3").setSource("name", "test3", "number", 3).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(
@@ -141,6 +142,7 @@ public void testSimpleMatchedQueryFromTopLevelFilter() throws Exception {
         client().prepareIndex("test").setId("2").setSource("name", "test").get();
         client().prepareIndex("test").setId("3").setSource("name", "test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -192,6 +194,7 @@ public void testSimpleMatchedQueryFromTopLevelFilterAndFilteredQuery() throws Ex
         client().prepareIndex("test").setId("2").setSource("name", "test", "title", "title2").get();
         client().prepareIndex("test").setId("3").setSource("name", "test", "title", "title3").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(boolQuery().must(matchAllQuery()).filter(termsQuery("title", "title1", "title2", "title3").queryName("title")))
@@ -224,12 +227,13 @@ public void testSimpleMatchedQueryFromTopLevelFilterAndFilteredQuery() throws Ex
         }
     }
 
-    public void testRegExpQuerySupportsName() {
+    public void testRegExpQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.regexpQuery("title", "title1").queryName("regex"))
@@ -246,12 +250,13 @@ public void testRegExpQuerySupportsName() {
         }
     }
 
-    public void testPrefixQuerySupportsName() {
+    public void testPrefixQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.prefixQuery("title", "title").queryName("prefix"))
@@ -268,12 +273,13 @@ public void testPrefixQuerySupportsName() {
         }
     }
 
-    public void testFuzzyQuerySupportsName() {
+    public void testFuzzyQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.fuzzyQuery("title", "titel1").queryName("fuzzy"))
@@ -290,12 +296,13 @@ public void testFuzzyQuerySupportsName() {
         }
     }
 
-    public void testWildcardQuerySupportsName() {
+    public void testWildcardQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.wildcardQuery("title", "titl*").queryName("wildcard"))
@@ -312,12 +319,13 @@ public void testWildcardQuerySupportsName() {
         }
     }
 
-    public void testSpanFirstQuerySupportsName() {
+    public void testSpanFirstQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1 title2").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.spanFirstQuery(QueryBuilders.spanTermQuery("title", "title1"), 10).queryName("span"))
@@ -344,6 +352,7 @@ public void testMatchedWithShould() throws Exception {
         client().prepareIndex("test").setId("1").setSource("content", "Lorem ipsum dolor sit amet").get();
         client().prepareIndex("test").setId("2").setSource("content", "consectetur adipisicing elit").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // Execute search at least two times to load it in cache
         int iter = scaledRandomIntBetween(2, 10);
@@ -378,6 +387,7 @@ public void testMatchedWithWrapperQuery() throws Exception {
 
         client().prepareIndex("test").setId("1").setSource("content", "Lorem ipsum dolor sit amet").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         MatchQueryBuilder matchQueryBuilder = matchQuery("content", "amet").queryName("abc");
         BytesReference matchBytes = XContentHelper.toXContent(matchQueryBuilder, MediaTypeRegistry.JSON, false);
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java
index 53bded1fc493c..d2bca41760ff6 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java
@@ -465,6 +465,7 @@ public void testQueryStringAnalyzedWildcard() throws Exception {
 
         client().prepareIndex("test").setId("1").setSource("field1", "value_1", "field2", "value_2").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("value*")).get();
         assertHitCount(searchResponse, 1L);
@@ -482,11 +483,12 @@ public void testQueryStringAnalyzedWildcard() throws Exception {
         assertHitCount(searchResponse, 1L);
     }
 
-    public void testLowercaseExpandedTerms() {
+    public void testLowercaseExpandedTerms() throws InterruptedException {
         createIndex("test");
 
         client().prepareIndex("test").setId("1").setSource("field1", "value_1", "field2", "value_2").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("VALUE_3~1")).get();
         assertHitCount(searchResponse, 1L);
@@ -499,7 +501,7 @@ public void testLowercaseExpandedTerms() {
     }
 
     // Issue #3540
-    public void testDateRangeInQueryString() {
+    public void testDateRangeInQueryString() throws InterruptedException {
         // the mapping needs to be provided upfront otherwise we are not sure how many failures we get back
         // as with dynamic mappings some shards might be lacking behind and parse a different query
         assertAcked(prepareCreate("test").setMapping("past", "type=date", "future", "type=date"));
@@ -510,6 +512,7 @@ public void testDateRangeInQueryString() {
         client().prepareIndex("test").setId("1").setSource("past", aMonthAgo, "future", aMonthFromNow).get();
         refresh();
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("past:[now-2M/d TO now/d]")).get();
         assertHitCount(searchResponse, 1L);
 
@@ -525,7 +528,7 @@ public void testDateRangeInQueryString() {
     }
 
     // Issue #7880
-    public void testDateRangeInQueryStringWithTimeZone_7880() {
+    public void testDateRangeInQueryStringWithTimeZone_7880() throws InterruptedException {
         // the mapping needs to be provided upfront otherwise we are not sure how many failures we get back
         // as with dynamic mappings some shards might be lacking behind and parse a different query
         assertAcked(prepareCreate("test").setMapping("past", "type=date"));
@@ -536,6 +539,7 @@ public void testDateRangeInQueryStringWithTimeZone_7880() {
         client().prepareIndex("test").setId("1").setSource("past", now).get();
         refresh();
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(queryStringQuery("past:[now-1m/m TO now+1m/m]").timeZone(timeZone.getId()))
             .get();
@@ -543,7 +547,7 @@ public void testDateRangeInQueryStringWithTimeZone_7880() {
     }
 
     // Issue #10477
-    public void testDateRangeInQueryStringWithTimeZone_10477() {
+    public void testDateRangeInQueryStringWithTimeZone_10477() throws InterruptedException {
         // the mapping needs to be provided upfront otherwise we are not sure how many failures we get back
         // as with dynamic mappings some shards might be lacking behind and parse a different query
         assertAcked(prepareCreate("test").setMapping("past", "type=date"));
@@ -552,6 +556,7 @@ public void testDateRangeInQueryStringWithTimeZone_10477() {
         client().prepareIndex("test").setId("2").setSource("past", "2015-04-06T00:00:00+0000").get();
         refresh();
 
+        indexRandomForConcurrentSearch("test");
         // Timezone set with dates
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(queryStringQuery("past:[2015-04-06T00:00:00+0200 TO 2015-04-06T23:00:00+0200]"))
@@ -725,6 +730,7 @@ public void testPassQueryOrFilterAsJSONString() throws Exception {
         createIndex("test");
 
         client().prepareIndex("test").setId("1").setSource("field1", "value1_1", "field2", "value2_1").setRefreshPolicy(IMMEDIATE).get();
+        indexRandomForConcurrentSearch("test");
 
         WrapperQueryBuilder wrapper = new WrapperQueryBuilder("{ \"term\" : { \"field1\" : \"value1_1\" } }");
         assertHitCount(client().prepareSearch().setQuery(wrapper).get(), 1L);
@@ -741,6 +747,7 @@ public void testFiltersWithCustomCacheKey() throws Exception {
 
         client().prepareIndex("test").setId("1").setSource("field1", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch("test").setQuery(constantScoreQuery(termsQuery("field1", "value1"))).get();
         assertHitCount(searchResponse, 1L);
 
@@ -782,6 +789,7 @@ public void testMatchQueryFuzzy() throws Exception {
             client().prepareIndex("test").setId("1").setSource("text", "Unit"),
             client().prepareIndex("test").setId("2").setSource("text", "Unity")
         );
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness(Fuzziness.ZERO)).get();
         assertHitCount(searchResponse, 0L);
@@ -913,11 +921,12 @@ public void testMultiMatchQueryZeroTermsQuery() {
         assertHitCount(searchResponse, 2L);
     }
 
-    public void testMultiMatchQueryMinShouldMatch() {
+    public void testMultiMatchQueryMinShouldMatch() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test").setId("1").setSource("field1", new String[] { "value1", "value2", "value3" }).get();
         client().prepareIndex("test").setId("2").setSource("field2", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         MultiMatchQueryBuilder multiMatchQuery = multiMatchQuery("value1 value2 foo", "field1", "field2");
 
@@ -959,12 +968,13 @@ public void testMultiMatchQueryMinShouldMatch() {
         assertHitCount(searchResponse, 0L);
     }
 
-    public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws IOException {
+    public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws IOException, InterruptedException {
         createIndex("test");
         client().prepareIndex("test").setId("1").setSource("field1", new String[] { "value1", "value2", "value3" }).get();
         client().prepareIndex("test").setId("2").setSource("field2", "value1").get();
         refresh();
 
+        indexRandomForConcurrentSearch("test");
         BoolQueryBuilder boolQuery = boolQuery().must(termQuery("field1", "value1"))
             .should(boolQuery().should(termQuery("field1", "value1")).should(termQuery("field1", "value2")).minimumShouldMatch(3));
         SearchResponse searchResponse = client().prepareSearch().setQuery(boolQuery).get();
@@ -991,12 +1001,13 @@ public void testBoolQueryMinShouldMatchBiggerThanNumberOfShouldClauses() throws
         assertHitCount(searchResponse, 0L);
     }
 
-    public void testFuzzyQueryString() {
+    public void testFuzzyQueryString() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test").setId("1").setSource("str", "foobar", "date", "2012-02-01", "num", 12).get();
         client().prepareIndex("test").setId("2").setSource("str", "fred", "date", "2012-02-05", "num", 20).get();
         refresh();
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("str:foobaz~1")).get();
         assertNoFailures(searchResponse);
         assertHitCount(searchResponse, 1L);
@@ -1015,6 +1026,7 @@ public void testQuotedQueryStringWithBoost() throws InterruptedException {
             client().prepareIndex("test").setId("2").setSource("important", "nothing important", "less_important", "phrase match")
         );
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(queryStringQuery("\"phrase match\"").field("important", boost).field("less_important"))
             .get();
@@ -1027,11 +1039,12 @@ public void testQuotedQueryStringWithBoost() throws InterruptedException {
         );
     }
 
-    public void testSpecialRangeSyntaxInQueryString() {
+    public void testSpecialRangeSyntaxInQueryString() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test").setId("1").setSource("str", "foobar", "date", "2012-02-01", "num", 12).get();
         client().prepareIndex("test").setId("2").setSource("str", "fred", "date", "2012-02-05", "num", 20).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(queryStringQuery("num:>19")).get();
         assertHitCount(searchResponse, 1L);
@@ -1137,6 +1150,7 @@ public void testTermsQuery() throws Exception {
 
     public void testTermsLookupFilter() throws Exception {
         assertAcked(prepareCreate("lookup").setMapping("terms", "type=text", "other", "type=text"));
+        indexRandomForConcurrentSearch("lookup");
         assertAcked(
             prepareCreate("lookup2").setMapping(
                 jsonBuilder().startObject()
@@ -1152,8 +1166,11 @@ public void testTermsLookupFilter() throws Exception {
                     .endObject()
             )
         );
+        indexRandomForConcurrentSearch("lookup2");
         assertAcked(prepareCreate("lookup3").setMapping("_source", "enabled=false", "terms", "type=text"));
+        indexRandomForConcurrentSearch("lookup3");
         assertAcked(prepareCreate("test").setMapping("term", "type=text"));
+        indexRandomForConcurrentSearch("test");
 
         indexRandom(
             true,
@@ -1279,6 +1296,7 @@ public void testBasicQueryById() throws Exception {
         client().prepareIndex("test").setId("3").setSource("field1", "value3").get();
         refresh();
 
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch().setQuery(idsQuery().addIds("1", "2")).get();
         assertHitCount(searchResponse, 2L);
         assertThat(searchResponse.getHits().getHits().length, equalTo(2));
@@ -1333,6 +1351,7 @@ public void testNumericTermsAndRanges() throws Exception {
             .setSource("num_byte", 17, "num_short", 17, "num_integer", 17, "num_long", 17, "num_float", 17, "num_double", 17)
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse;
         logger.info("--> term query on 1");
@@ -1439,6 +1458,7 @@ public void testNumericRangeFilter_2826() throws Exception {
         client().prepareIndex("test").setId("3").setSource("field1", "test2", "num_long", 3).get();
         client().prepareIndex("test").setId("4").setSource("field1", "test2", "num_long", 4).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setPostFilter(boolQuery().should(rangeQuery("num_long").from(1).to(2)).should(rangeQuery("num_long").from(3).to(4)))
@@ -1535,7 +1555,7 @@ public void testSimpleSpan() throws IOException, ExecutionException, Interrupted
         assertHitCount(searchResponse, 3L);
     }
 
-    public void testSpanMultiTermQuery() throws IOException {
+    public void testSpanMultiTermQuery() throws IOException, InterruptedException {
         createIndex("test");
 
         client().prepareIndex("test").setId("1").setSource("description", "foo other anything bar", "count", 1).get();
@@ -1543,6 +1563,7 @@ public void testSpanMultiTermQuery() throws IOException {
         client().prepareIndex("test").setId("3").setSource("description", "foo other", "count", 3).get();
         client().prepareIndex("test").setId("4").setSource("description", "fop", "count", 4).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setQuery(spanOrQuery(spanMultiTermQueryBuilder(fuzzyQuery("description", "fop"))))
@@ -1574,6 +1595,7 @@ public void testSpanNot() throws IOException, ExecutionException, InterruptedExc
         client().prepareIndex("test").setId("1").setSource("description", "the quick brown fox jumped over the lazy dog").get();
         client().prepareIndex("test").setId("2").setSource("description", "the quick black fox leaped over the sleeping dog").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(
@@ -1612,7 +1634,7 @@ public void testSpanNot() throws IOException, ExecutionException, InterruptedExc
         assertHitCount(searchResponse, 1L);
     }
 
-    public void testSimpleDFSQuery() throws IOException {
+    public void testSimpleDFSQuery() throws IOException, InterruptedException {
         assertAcked(
             prepareCreate("test").setMapping(
                 jsonBuilder().startObject()
@@ -1657,6 +1679,7 @@ public void testSimpleDFSQuery() throws IOException {
             .setSource("online", true, "ts", System.currentTimeMillis() - 123123, "type", "bs")
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
@@ -1679,8 +1702,9 @@ public void testSimpleDFSQuery() throws IOException {
         assertNoFailures(response);
     }
 
-    public void testMultiFieldQueryString() {
+    public void testMultiFieldQueryString() throws InterruptedException {
         client().prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").setRefreshPolicy(IMMEDIATE).get();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("regular");
         assertHitCount(client().prepareSearch("test").setQuery(queryStringQuery("value1").field("field1").field("field2")).get(), 1);
@@ -1700,11 +1724,12 @@ public void testMultiFieldQueryString() {
     }
 
     // see #3797
-    public void testMultiMatchLenientIssue3797() {
+    public void testMultiMatchLenientIssue3797() throws InterruptedException {
         createIndex("test");
 
         client().prepareIndex("test").setId("1").setSource("field1", 123, "field2", "value2").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(multiMatchQuery("value2", "field2").field("field1", 2).lenient(true))
@@ -1728,6 +1753,7 @@ public void testMinScore() throws ExecutionException, InterruptedException {
         client().prepareIndex("test").setId("3").setSource("score", 2.0).get();
         client().prepareIndex("test").setId("4").setSource("score", 0.5).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(functionScoreQuery(ScoreFunctionBuilders.fieldValueFactorFunction("score").missing(1.0)).setMinScore(1.5f))
@@ -1737,12 +1763,13 @@ public void testMinScore() throws ExecutionException, InterruptedException {
         assertSecondHit(searchResponse, hasId("1"));
     }
 
-    public void testQueryStringWithSlopAndFields() {
+    public void testQueryStringWithSlopAndFields() throws InterruptedException {
         assertAcked(prepareCreate("test"));
 
         client().prepareIndex("test").setId("1").setSource("desc", "one two three", "type", "customer").get();
         client().prepareIndex("test").setId("2").setSource("desc", "one two three", "type", "product").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         {
             SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(QueryBuilders.queryStringQuery("\"one two\"").defaultField("desc"))
@@ -1809,6 +1836,7 @@ public void testRangeQueryWithTimeZone() throws Exception {
                 .setId("4")
                 .setSource("date", Instant.now().atZone(ZoneOffset.ofHours(1)).toInstant().toEpochMilli(), "num", 4)
         );
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(QueryBuilders.rangeQuery("date").from("2014-01-01T00:00:00").to("2014-01-01T00:59:00"))
@@ -1948,6 +1976,7 @@ public void testMatchPhrasePrefixQuery() throws ExecutionException, InterruptedE
     public void testQueryStringParserCache() throws Exception {
         createIndex("test");
         indexRandom(true, false, client().prepareIndex("test").setId("1").setSource("nameTokens", "xyz"));
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test")
             .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
@@ -1978,6 +2007,7 @@ public void testRangeQueryRangeFields_24744() throws Exception {
             .setSource(jsonBuilder().startObject().startObject("int_range").field("gte", 10).field("lte", 20).endObject().endObject())
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         RangeQueryBuilder range = new RangeQueryBuilder("int_range").relation("intersects").from(Integer.MIN_VALUE).to(Integer.MAX_VALUE);
         SearchResponse searchResponse = client().prepareSearch("test").setQuery(range).get();
@@ -2013,6 +2043,7 @@ public void testNestedQueryWithFieldAlias() throws Exception {
 
         index("index", "_doc", "1", source);
         refresh();
+        indexRandomForConcurrentSearch("index");
 
         QueryBuilder nestedQuery = QueryBuilders.nestedQuery(
             "section",
@@ -2041,6 +2072,7 @@ public void testFieldAliasesForMetaFields() throws Exception {
 
         IndexRequestBuilder indexRequest = client().prepareIndex("test").setId("1").setRouting("custom").setSource("field", "value");
         indexRandom(true, false, indexRequest);
+        indexRandomForConcurrentSearch("test");
         client().admin()
             .cluster()
             .prepareUpdateSettings()
@@ -2073,7 +2105,7 @@ public void testFieldAliasesForMetaFields() throws Exception {
     /**
     * Test that wildcard queries on keyword fields get normalized
     */
-    public void testWildcardQueryNormalizationOnKeywordField() {
+    public void testWildcardQueryNormalizationOnKeywordField() throws InterruptedException {
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder()
@@ -2084,6 +2116,7 @@ public void testWildcardQueryNormalizationOnKeywordField() {
         );
         client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         {
             WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
@@ -2099,7 +2132,7 @@ public void testWildcardQueryNormalizationOnKeywordField() {
     /**
      * Test that wildcard queries on text fields get normalized
      */
-    public void testWildcardQueryNormalizationOnTextField() {
+    public void testWildcardQueryNormalizationOnTextField() throws InterruptedException {
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder()
@@ -2111,6 +2144,7 @@ public void testWildcardQueryNormalizationOnTextField() {
         );
         client().prepareIndex("test").setId("1").setSource("field1", "Bbb Aaa").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         {
             // test default case insensitivity: false
@@ -2130,10 +2164,11 @@ public void testWildcardQueryNormalizationOnTextField() {
     }
 
     /** tests wildcard case sensitivity */
-    public void testWildcardCaseSensitivity() {
+    public void testWildcardCaseSensitivity() throws InterruptedException {
         assertAcked(prepareCreate("test").setMapping("field", "type=text"));
         client().prepareIndex("test").setId("1").setSource("field", "lowercase text").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // test case sensitive
         SearchResponse response = client().prepareSearch("test").setQuery(wildcardQuery("field", "Text").caseInsensitive(false)).get();
@@ -2151,7 +2186,7 @@ public void testWildcardCaseSensitivity() {
      * Reserved characters should be excluded when the normalization is applied for keyword fields.
      * See https://github.com/elastic/elasticsearch/issues/46300 for details.
      */
-    public void testWildcardQueryNormalizationKeywordSpecialCharacters() {
+    public void testWildcardQueryNormalizationKeywordSpecialCharacters() throws InterruptedException {
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder()
@@ -2163,6 +2198,7 @@ public void testWildcardQueryNormalizationKeywordSpecialCharacters() {
         );
         client().prepareIndex("test").setId("1").setSource("field", "label-1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         WildcardQueryBuilder wildCardQuery = wildcardQuery("field", "la*");
         SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
@@ -2213,11 +2249,12 @@ public Map> getTokenizers() {
      * set for fuzzy queries with "constant_score" rewrite nested inside a `span_multi` query and would cause NPEs due to an unset
      * {@link AttributeSource}.
      */
-    public void testIssueFuzzyInsideSpanMulti() {
+    public void testIssueFuzzyInsideSpanMulti() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test").setId("1").setSource("field", "foobarbaz").get();
         ensureGreen();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         BoolQueryBuilder query = boolQuery().filter(spanMultiTermQueryBuilder(fuzzyQuery("field", "foobarbiz").rewrite("constant_score")));
         SearchResponse response = client().prepareSearch("test").setQuery(query).get();

From 39308bf5d962a70fe0e3103a354899ce506562e6 Mon Sep 17 00:00:00 2001
From: Ethan Emoto <70109490+e-emoto@users.noreply.github.com>
Date: Tue, 14 Nov 2023 10:34:29 -0800
Subject: [PATCH 294/388] Fixed test cases for concurrent search (#11184)

Updated flaky and failing test cases after changes to indexRandomForConcurrentSearch
	    Remove unnecessary call to indexRandomForConcurrentSearch in testSearchAndRelocateConcurrently

Signed-off-by: Ethan Emoto 
---
 .../org/opensearch/search/fetch/FetchSubPhasePluginIT.java    | 4 ----
 .../search/functionscore/RandomScoreFunctionIT.java           | 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java
index 77ddc5bf636f0..87f2153eb800f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java
@@ -100,10 +100,6 @@ protected Collection> nodePlugins() {
 
     @SuppressWarnings("unchecked")
     public void testPlugin() throws Exception {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11112",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         client().admin()
             .indices()
             .prepareCreate("test")
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java
index 54cfcb7e6a683..69e30fc879dd8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java
@@ -135,6 +135,7 @@ public void testConsistentHitsWithSameSeed() throws Exception {
         }
         flush();
         refresh();
+        indexRandomForConcurrentSearch("test");
         int outerIters = scaledRandomIntBetween(10, 20);
         for (int o = 0; o < outerIters; o++) {
             final int seed = randomInt();
@@ -299,6 +300,7 @@ public void testSeedReportedInExplain() throws Exception {
         index("test", "type", "1", jsonBuilder().startObject().endObject());
         flush();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         int seed = 12345678;
 
@@ -318,6 +320,7 @@ public void testSeedAndNameReportedInExplain() throws Exception {
         index("test", "type", "1", jsonBuilder().startObject().endObject());
         flush();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         int seed = 12345678;
 

From 3ff7e9710ab4c4625d7a8100aab0451e6f9c8d13 Mon Sep 17 00:00:00 2001
From: Jay Deng 
Date: Tue, 14 Nov 2023 11:10:26 -0800
Subject: [PATCH 295/388] Add indexRandomForConcurrentSearch for tests:
 (#11181)

* MultiSearchIT.java
* SimpleNestedIT.java
* SearchPreferenceIT.java
* SimpleQueryStringIT.java
* ScriptQuerySearchIT.java

Signed-off-by: Jay Deng 
---
 .../search/msearch/MultiSearchIT.java         |  6 ++--
 .../search/nested/SimpleNestedIT.java         | 28 ++++++++++++++++++-
 .../search/preference/SearchPreferenceIT.java |  7 +++--
 .../search/query/SimpleQueryStringIT.java     |  1 +
 .../scriptfilter/ScriptQuerySearchIT.java     |  5 +++-
 5 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java
index bc1d2833ecbbf..b35208941d2a2 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java
@@ -71,12 +71,13 @@ protected Settings featureFlagSettings() {
         return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
     }
 
-    public void testSimpleMultiSearch() {
+    public void testSimpleMultiSearch() throws InterruptedException {
         createIndex("test");
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("field", "xxx").get();
         client().prepareIndex("test").setId("2").setSource("field", "yyy").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         MultiSearchResponse response = client().prepareMultiSearch()
             .add(client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "xxx")))
             .add(client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "yyy")))
@@ -94,13 +95,14 @@ public void testSimpleMultiSearch() {
         assertFirstHit(response.getResponses()[1].getResponse(), hasId("2"));
     }
 
-    public void testSimpleMultiSearchMoreRequests() {
+    public void testSimpleMultiSearchMoreRequests() throws InterruptedException {
         createIndex("test");
         int numDocs = randomIntBetween(0, 16);
         for (int i = 0; i < numDocs; i++) {
             client().prepareIndex("test").setId(Integer.toString(i)).setSource("{}", MediaTypeRegistry.JSON).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         int numSearchRequests = randomIntBetween(1, 64);
         MultiSearchRequest request = new MultiSearchRequest();
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java
index 656e7b2e366ed..8b375841c2913 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java
@@ -126,6 +126,7 @@ public void testSimpleNested() throws Exception {
             .get();
 
         waitForRelocation(ClusterHealthStatus.GREEN);
+        indexRandomForConcurrentSearch("test");
         GetResponse getResponse = client().prepareGet("test", "1").get();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getSourceAsBytes(), notNullValue());
@@ -500,6 +501,10 @@ public void testExplain() throws Exception {
     }
 
     public void testSimpleNestedSorting() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11187",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put("index.refresh_interval", -1))
                 .setMapping(
@@ -569,6 +574,7 @@ public void testSimpleNestedSorting() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchAllQuery())
@@ -598,6 +604,10 @@ public void testSimpleNestedSorting() throws Exception {
     }
 
     public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11187",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put("index.refresh_interval", -1))
                 .setMapping(
@@ -677,6 +687,7 @@ public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test")
             .setQuery(QueryBuilders.matchAllQuery())
@@ -729,6 +740,10 @@ public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception {
     }
 
     public void testNestedSortWithMultiLevelFiltering() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11187",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setMapping(
                 "{\n"
@@ -865,6 +880,7 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // access id = 1, read, max value, asc, should use grault and quxx
         SearchResponse searchResponse = client().prepareSearch()
@@ -971,7 +987,7 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception {
     // https://github.com/elastic/elasticsearch/issues/31554
     public void testLeakingSortValues() throws Exception {
         assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11065",
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11187",
             internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
         );
         assertAcked(
@@ -1063,6 +1079,10 @@ public void testLeakingSortValues() throws Exception {
     }
 
     public void testSortNestedWithNestedFilter() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11187",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setMapping(
                 XContentFactory.jsonBuilder()
@@ -1222,6 +1242,7 @@ public void testSortNestedWithNestedFilter() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // Without nested filter
         SearchResponse searchResponse = client().prepareSearch()
@@ -1460,6 +1481,10 @@ public void testSortNestedWithNestedFilter() throws Exception {
 
     // Issue #9305
     public void testNestedSortingWithNestedFilterAsFilter() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11187",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setMapping(
                 jsonBuilder().startObject()
@@ -1602,6 +1627,7 @@ public void testNestedSortingWithNestedFilterAsFilter() throws Exception {
             .get();
         assertTrue(indexResponse2.getShardInfo().getSuccessful() > 0);
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .addSort(SortBuilders.fieldSort("users.first").setNestedPath("users").order(SortOrder.ASC))
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java
index 97fe05f5b9747..6e40c08ed08a1 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java
@@ -160,12 +160,13 @@ public void testNoPreferenceRandom() throws Exception {
         assertThat(firstNodeId, not(equalTo(secondNodeId)));
     }
 
-    public void testSimplePreference() {
+    public void testSimplePreference() throws InterruptedException {
         client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", MediaTypeRegistry.JSON).get();
         ensureGreen();
 
         client().prepareIndex("test").setSource("field1", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).get();
         assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
@@ -264,7 +265,7 @@ private void assertSearchOnRandomNodes(SearchRequestBuilder request) {
         assertThat(hitNodes.size(), greaterThan(1));
     }
 
-    public void testCustomPreferenceUnaffectedByOtherShardMovements() {
+    public void testCustomPreferenceUnaffectedByOtherShardMovements() throws InterruptedException {
 
         /*
          * Custom preferences can be used to encourage searches to go to a consistent set of shard copies, meaning that other copies' data
@@ -283,6 +284,7 @@ public void testCustomPreferenceUnaffectedByOtherShardMovements() {
         ensureGreen();
         client().prepareIndex("test").setSource("field1", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         final String customPreference = randomAlphaOfLength(10);
 
@@ -302,6 +304,7 @@ public void testCustomPreferenceUnaffectedByOtherShardMovements() {
             prepareCreate("test2").setSettings(Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, replicasInNewIndex))
         );
         ensureGreen();
+        indexRandomForConcurrentSearch("test2");
 
         assertSearchesSpecificNode("test", customPreference, nodeId);
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
index 017d28ef3a2a6..d8902238005da 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
@@ -437,6 +437,7 @@ public void testSimpleQueryStringOnIndexMetaField() throws Exception {
         client().prepareIndex("test").setId("2").setSource("foo", 234, "bar", "bcd").get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("test").field("_index")).get();
         assertHitCount(searchResponse, 2L);
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java
index 34967528f2c4f..ae00904f237a5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/scriptfilter/ScriptQuerySearchIT.java
@@ -161,6 +161,7 @@ public void testCustomScriptBinaryField() throws Exception {
             .get();
         flush();
         refresh();
+        indexRandomForConcurrentSearch("my-index");
 
         SearchResponse response = client().prepareSearch()
             .setQuery(
@@ -213,6 +214,7 @@ public void testCustomScriptBoost() throws Exception {
             .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject())
             .get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         logger.info("running doc['num1'].value > 1");
         SearchResponse response = client().prepareSearch()
@@ -259,7 +261,7 @@ public void testCustomScriptBoost() throws Exception {
         assertThat(response.getHits().getAt(2).getFields().get("sNum1").getValues().get(0), equalTo(3.0));
     }
 
-    public void testDisallowExpensiveQueries() {
+    public void testDisallowExpensiveQueries() throws InterruptedException {
         try {
             assertAcked(prepareCreate("test-index").setMapping("num1", "type=double"));
             int docCount = 10;
@@ -267,6 +269,7 @@ public void testDisallowExpensiveQueries() {
                 client().prepareIndex("test-index").setId("" + i).setSource("num1", i).get();
             }
             refresh();
+            indexRandomForConcurrentSearch("test-index");
 
             // Execute with search.allow_expensive_queries = null => default value = false => success
             Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > 1", Collections.emptyMap());

From 865766916ba04f6b5eeaf209a4b234b190708cca Mon Sep 17 00:00:00 2001
From: Ganesh Krishna Ramadurai 
Date: Tue, 14 Nov 2023 12:00:48 -0800
Subject: [PATCH 296/388] Fix tests for concurrent search (#11130)

Signed-off-by: Ganesh Ramadurai 
Signed-off-by: Ganesh Ramadurai 
Co-authored-by: Ganesh Ramadurai 
---
 .../org/opensearch/search/scroll/SearchScrollIT.java   |  3 ++-
 .../java/org/opensearch/search/sort/FieldSortIT.java   |  5 +++++
 .../java/org/opensearch/search/sort/GeoDistanceIT.java |  3 ++-
 .../org/opensearch/search/sort/SortFromPluginIT.java   |  2 ++
 .../opensearch/search/source/MetadataFetchingIT.java   | 10 ++++++----
 .../org/opensearch/search/source/SourceFetchingIT.java |  3 ++-
 .../org/opensearch/search/suggest/SuggestSearchIT.java |  1 +
 7 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java
index 0eee136acac69..b2b6409580061 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java
@@ -733,7 +733,7 @@ public void testInvalidScrollKeepAlive() throws IOException {
      * Ensures that we always create and retain search contexts on every target shards for a scroll request
      * regardless whether that query can be written to match_no_docs on some target shards or not.
      */
-    public void testScrollRewrittenToMatchNoDocs() {
+    public void testScrollRewrittenToMatchNoDocs() throws InterruptedException {
         final int numShards = randomIntBetween(3, 5);
         assertAcked(
             client().admin()
@@ -746,6 +746,7 @@ public void testScrollRewrittenToMatchNoDocs() {
         client().prepareIndex("test").setId("2").setSource("created_date", "2020-01-02").get();
         client().prepareIndex("test").setId("3").setSource("created_date", "2020-01-03").get();
         client().admin().indices().prepareRefresh("test").get();
+        indexRandomForConcurrentSearch("test");
         SearchResponse resp = null;
         try {
             int totalHits = 0;
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
index d4980a64a3977..9c788110af937 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
@@ -265,6 +265,10 @@ public void testIssue6614() throws ExecutionException, InterruptedException {
     }
 
     public void testTrackScores() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11189",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(client().admin().indices().prepareCreate("test").setMapping("svalue", "type=keyword").get());
         ensureGreen();
         index(
@@ -278,6 +282,7 @@ public void testTrackScores() throws Exception {
             jsonBuilder().startObject().field("id", "2").field("svalue", "bbb").field("ivalue", 200).field("dvalue", 0.2).endObject()
         );
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addSort("svalue", SortOrder.ASC).get();
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java
index 6886f8d67589e..ff24a1f962332 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java
@@ -602,7 +602,7 @@ public void testDistanceSortingNestedFields() throws Exception {
     /**
      * Issue 3073
      */
-    public void testGeoDistanceFilter() throws IOException {
+    public void testGeoDistanceFilter() throws IOException, InterruptedException {
         Version version = VersionUtils.randomIndexCompatibleVersion(random());
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
         double lat = 40.720611;
@@ -620,6 +620,7 @@ public void testGeoDistanceFilter() throws IOException {
         assertAcked(prepareCreate("locations").setSettings(settings).setMapping(mapping));
         client().prepareIndex("locations").setId("1").setCreate(true).setSource(source).get();
         refresh();
+        indexRandomForConcurrentSearch("locations");
         client().prepareGet("locations", "1").get();
 
         SearchResponse result = client().prepareSearch("locations")
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java
index 7bcded86fcaa8..76e68781c72ba 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/SortFromPluginIT.java
@@ -59,6 +59,7 @@ public void testPluginSort() throws Exception {
         client().prepareIndex("test").setId("3").setSource("field", 0).get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test").addSort(new CustomSortBuilder("field", SortOrder.ASC)).get();
         assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("3"));
@@ -80,6 +81,7 @@ public void testPluginSortXContent() throws Exception {
         client().prepareIndex("test").setId("3").setSource("field", 0).get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // builder -> json -> builder
         SearchResponse searchResponse = client().prepareSearch("test")
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java
index c98a38ea0bb97..a9c4bf841d9a1 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/source/MetadataFetchingIT.java
@@ -76,12 +76,13 @@ protected Settings featureFlagSettings() {
         return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
     }
 
-    public void testSimple() {
+    public void testSimple() throws InterruptedException {
         assertAcked(prepareCreate("test"));
         ensureGreen();
 
         client().prepareIndex("test").setId("1").setSource("field", "value").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test").storedFields("_none_").setFetchSource(false).setVersion(true).get();
         assertThat(response.getHits().getAt(0).getId(), nullValue());
@@ -93,12 +94,12 @@ public void testSimple() {
         assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue());
     }
 
-    public void testInnerHits() {
+    public void testInnerHits() throws InterruptedException {
         assertAcked(prepareCreate("test").setMapping("nested", "type=nested"));
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("field", "value", "nested", Collections.singletonMap("title", "foo")).get();
         refresh();
-
+        indexRandomForConcurrentSearch("test");
         SearchResponse response = client().prepareSearch("test")
             .storedFields("_none_")
             .setFetchSource(false)
@@ -119,12 +120,13 @@ public void testInnerHits() {
         assertThat(hits.getAt(0).getSourceAsString(), nullValue());
     }
 
-    public void testWithRouting() {
+    public void testWithRouting() throws InterruptedException {
         assertAcked(prepareCreate("test"));
         ensureGreen();
 
         client().prepareIndex("test").setId("1").setSource("field", "value").setRouting("toto").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test").storedFields("_none_").setFetchSource(false).get();
         assertThat(response.getHits().getAt(0).getId(), nullValue());
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java
index eeef5403fe898..0e9932051141b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java
@@ -66,12 +66,13 @@ protected Settings featureFlagSettings() {
         return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
     }
 
-    public void testSourceDefaultBehavior() {
+    public void testSourceDefaultBehavior() throws InterruptedException {
         createIndex("test");
         ensureGreen();
 
         index("test", "type1", "1", "field", "value");
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue());
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java
index 32bb0e34054bb..dd6179d3820be 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java
@@ -287,6 +287,7 @@ public void testSizeOneShard() throws Exception {
             index("test", "type1", Integer.toString(i), "text", "abc" + i);
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellchecker")).get();
         assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());

From 675dd416156a628307f7bad0a5ea8794f9d9b942 Mon Sep 17 00:00:00 2001
From: Wei Wang <93847013+weiwang118@users.noreply.github.com>
Date: Tue, 14 Nov 2023 16:56:04 -0800
Subject: [PATCH 297/388] Fix tests for concurrent Search: TermsDocCountErrorIT
 (#11093)

Signed-off-by: Wei Wang 
Co-authored-by: Wei Wang 
---
 .../search/aggregations/bucket/TermsDocCountErrorIT.java         | 1 +
 .../opensearch/test/ParameterizedOpenSearchIntegTestCase.java    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java
index 3fcf4b5d533d4..b355ce6d7a8dd 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsDocCountErrorIT.java
@@ -225,6 +225,7 @@ public void setupSuiteScopeCluster() throws Exception {
         }
 
         indexRandom(true, builders);
+        indexRandomForMultipleSlices("idx");
         ensureSearchable();
     }
 
diff --git a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java
index f8813a8c5afa9..edda6bf5603f7 100644
--- a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java
@@ -47,6 +47,7 @@ public void afterTests() {
         client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settingsToUnset).get();
     }
 
+    // This method shouldn't be called in setupSuiteScopeCluster(). Only call this method inside single test.
     public void indexRandomForConcurrentSearch(String... indices) throws InterruptedException {
         if (dynamicSettings.get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey()).equals("true")) {
             indexRandomForMultipleSlices(indices);

From c1866d9e70ef87d69cc6c49b3234ac5269c43a28 Mon Sep 17 00:00:00 2001
From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com>
Date: Wed, 15 Nov 2023 20:49:21 +0530
Subject: [PATCH 298/388] Skipping half_float sort tests for 2.11.0 as well
 (#11214)

Signed-off-by: Chaitanya Gohel 
---
 .../resources/rest-api-spec/test/search/260_sort_mixed.yml    | 4 ++--
 .../resources/rest-api-spec/test/search/90_search_after.yml   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml
index a04dc308b2a06..7385eef051243 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_mixed.yml
@@ -124,8 +124,8 @@
 ---
 "search across indices with mixed long and double and float numeric types":
   - skip:
-      version: " - 2.10.99"
-      reason: half float was broken before 2.11
+      version: " - 2.11.0"
+      reason: half float was broken before 2.11.1
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml
index 1563daba9de6d..41c8626afcd75 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml
@@ -324,8 +324,8 @@
 ---
 "half float":
   - skip:
-        version: " - 2.10.99"
-        reason: half_float was broken for 2.10 and earlier
+        version: " - 2.11.0"
+        reason: half_float was broken for 2.11.0 and earlier
 
   - do:
       indices.create:

From 379acf30cce5fe3b38bbd806183bf06f6937450a Mon Sep 17 00:00:00 2001
From: David Z <38449481+dzane17@users.noreply.github.com>
Date: Wed, 15 Nov 2023 09:19:28 -0800
Subject: [PATCH 299/388] Per request latency onPhaseEnd fix (#10934)

Signed-off-by: David Zane 
---
 CHANGELOG.md                                                    | 1 +
 .../org/opensearch/action/search/AbstractSearchAsyncAction.java | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2165fcb4700fb..99c31f00278b8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -147,6 +147,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix passing wrong parameter when calling newConfigurationException() in DotExpanderProcessor ([#10737](https://github.com/opensearch-project/OpenSearch/pull/10737))
 - Fix SuggestSearch.testSkipDuplicates by forceing refresh when indexing its test documents ([#11068](https://github.com/opensearch-project/OpenSearch/pull/11068))
 - Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095))
+- Fix per request latency last phase not tracked ([#10934](https://github.com/opensearch-project/OpenSearch/pull/10934))
 
 ### Security
 
diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
index 14f57218ae1dc..d27bc931e827b 100644
--- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
@@ -696,9 +696,9 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At
                     searchContextId = null;
                 }
             }
+            onPhaseEnd();
             listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId));
         }
-        onPhaseEnd();
         setCurrentPhase(null);
     }
 

From c0c76e6db6b6a39401700de54f7aa64e2cc6f9f1 Mon Sep 17 00:00:00 2001
From: Martin Gaievski 
Date: Wed, 15 Nov 2023 12:01:47 -0800
Subject: [PATCH 300/388] Passing IndicesService to ingest processor factory
 with processor params (#10307)

* Passing IngestService to processor factory with processor params

Signed-off-by: Martin Gaievski 

* Fixed typo in the changlelog entry for this PR

Signed-off-by: Martin Gaievski 

---------

Signed-off-by: Martin Gaievski 
---
 CHANGELOG.md                                  |  1 +
 .../org/opensearch/ingest/IngestService.java  |  7 +++++--
 .../java/org/opensearch/ingest/Processor.java |  7 ++++++-
 .../main/java/org/opensearch/node/Node.java   | 20 ++++++++++---------
 .../opensearch/ingest/IngestServiceTests.java | 15 +++++++++-----
 .../snapshots/SnapshotResiliencyTests.java    |  3 ++-
 6 files changed, 35 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 99c31f00278b8..202e0fa910c55 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -103,6 +103,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
 - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069))
 - [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672))
+- Allowing pipeline processors to access index mapping info by passing ingest service ref as part of the processor factory parameters ([#10307](https://github.com/opensearch-project/OpenSearch/pull/10307))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java
index baf357a4bc0d5..2d4439e86461b 100644
--- a/server/src/main/java/org/opensearch/ingest/IngestService.java
+++ b/server/src/main/java/org/opensearch/ingest/IngestService.java
@@ -72,6 +72,7 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.VersionType;
 import org.opensearch.index.analysis.AnalysisRegistry;
+import org.opensearch.indices.IndicesService;
 import org.opensearch.plugins.IngestPlugin;
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.ThreadPool;
@@ -128,7 +129,8 @@ public IngestService(
         ScriptService scriptService,
         AnalysisRegistry analysisRegistry,
         List ingestPlugins,
-        Client client
+        Client client,
+        IndicesService indicesService
     ) {
         this.clusterService = clusterService;
         this.scriptService = scriptService;
@@ -143,7 +145,8 @@ public IngestService(
                 (delay, command) -> threadPool.schedule(command, TimeValue.timeValueMillis(delay), ThreadPool.Names.GENERIC),
                 this,
                 client,
-                threadPool.generic()::execute
+                threadPool.generic()::execute,
+                indicesService
             )
         );
         this.threadPool = threadPool;
diff --git a/server/src/main/java/org/opensearch/ingest/Processor.java b/server/src/main/java/org/opensearch/ingest/Processor.java
index 6097045a87e21..ecae1c139ea5e 100644
--- a/server/src/main/java/org/opensearch/ingest/Processor.java
+++ b/server/src/main/java/org/opensearch/ingest/Processor.java
@@ -36,6 +36,7 @@
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.env.Environment;
 import org.opensearch.index.analysis.AnalysisRegistry;
+import org.opensearch.indices.IndicesService;
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.Scheduler;
 
@@ -156,6 +157,8 @@ class Parameters {
          */
         public final Client client;
 
+        public final IndicesService indicesService;
+
         public Parameters(
             Environment env,
             ScriptService scriptService,
@@ -165,7 +168,8 @@ public Parameters(
             BiFunction scheduler,
             IngestService ingestService,
             Client client,
-            Consumer genericExecutor
+            Consumer genericExecutor,
+            IndicesService indicesService
         ) {
             this.env = env;
             this.scriptService = scriptService;
@@ -176,6 +180,7 @@ public Parameters(
             this.ingestService = ingestService;
             this.client = client;
             this.genericExecutor = genericExecutor;
+            this.indicesService = indicesService;
         }
 
     }
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index e80b768074fc7..8fbb1a9306c6c 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -629,15 +629,6 @@ protected Node(
             metricsRegistry = metricsRegistryFactory.getMetricsRegistry();
             resourcesToClose.add(tracer::close);
             resourcesToClose.add(metricsRegistry::close);
-            final IngestService ingestService = new IngestService(
-                clusterService,
-                threadPool,
-                this.environment,
-                scriptService,
-                analysisModule.getAnalysisRegistry(),
-                pluginsService.filterPlugins(IngestPlugin.class),
-                client
-            );
 
             final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client);
             final UsageService usageService = new UsageService();
@@ -823,6 +814,17 @@ protected Node(
                 recoverySettings
             );
 
+            final IngestService ingestService = new IngestService(
+                clusterService,
+                threadPool,
+                this.environment,
+                scriptService,
+                analysisModule.getAnalysisRegistry(),
+                pluginsService.filterPlugins(IngestPlugin.class),
+                client,
+                indicesService
+            );
+
             final AliasValidator aliasValidator = new AliasValidator();
 
             final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService, systemIndices);
diff --git a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
index 1dfd41e96817e..2edfe87387c92 100644
--- a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
@@ -67,6 +67,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.VersionType;
+import org.opensearch.indices.IndicesService;
 import org.opensearch.plugins.IngestPlugin;
 import org.opensearch.script.MockScriptEngine;
 import org.opensearch.script.Script;
@@ -149,7 +150,8 @@ public void testIngestPlugin() {
             null,
             null,
             Collections.singletonList(DUMMY_PLUGIN),
-            client
+            client,
+            mock(IndicesService.class)
         );
         Map factories = ingestService.getProcessorFactories();
         assertTrue(factories.containsKey("foo"));
@@ -167,7 +169,8 @@ public void testIngestPluginDuplicate() {
                 null,
                 null,
                 Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN),
-                client
+                client,
+                mock(IndicesService.class)
             )
         );
         assertTrue(e.getMessage(), e.getMessage().contains("already registered"));
@@ -182,7 +185,8 @@ public void testExecuteIndexPipelineDoesNotExist() {
             null,
             null,
             Collections.singletonList(DUMMY_PLUGIN),
-            client
+            client,
+            mock(IndicesService.class)
         );
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
             .source(emptyMap())
@@ -1485,7 +1489,8 @@ public Map getProcessors(Processor.Parameters paramet
             null,
             null,
             Arrays.asList(testPlugin),
-            client
+            client,
+            mock(IndicesService.class)
         );
         ingestService.addIngestClusterStateListener(ingestClusterStateListener);
 
@@ -1702,7 +1707,7 @@ private static IngestService createWithProcessors(Map
             public Map getProcessors(final Processor.Parameters parameters) {
                 return processors;
             }
-        }), client);
+        }), client, mock(IndicesService.class));
     }
 
     private CompoundProcessor mockCompoundProcessor() {
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
index b7a2baacba611..352eeb779599c 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
@@ -2209,7 +2209,8 @@ public void onFailure(final Exception e) {
                             scriptService,
                             new AnalysisModule(environment, Collections.emptyList()).getAnalysisRegistry(),
                             Collections.emptyList(),
-                            client
+                            client,
+                            indicesService
                         ),
                         transportShardBulkAction,
                         client,

From 08db961e1914d93ce9d81e6da84e6e232b8d37e2 Mon Sep 17 00:00:00 2001
From: Ketan Verma <9292653+ketanv3@users.noreply.github.com>
Date: Thu, 16 Nov 2023 03:28:42 +0530
Subject: [PATCH 301/388] Refactor common parts from the Rounding class into a
 separate 'round' package (#11023)

* Refactor common parts from the Rounding class into a separate 'round' package

Signed-off-by: Ketan Verma 

* Move RoundableTests from :server to :libs:opensearch-common module

Signed-off-by: Ketan Verma 

* Address PR comments

Signed-off-by: Ketan Verma 

* Replace assert with IllegalArgumentException for size checks

Signed-off-by: Ketan Verma 

---------

Signed-off-by: Ketan Verma 
---
 CHANGELOG.md                                  |   1 +
 .../RoundableBenchmark.java}                  |  16 +-
 .../round/BidirectionalLinearSearcher.java    |  59 +++++++
 .../common/round/BinarySearcher.java          |  43 +++++
 .../opensearch/common/round/Roundable.java    |  28 ++++
 .../common/round/RoundableFactory.java        |  39 +++++
 .../opensearch/common/round/package-info.java |  12 ++
 .../common/round/RoundableTests.java          |  57 +++++++
 .../java/org/opensearch/common/Rounding.java  | 148 ++++--------------
 .../org/opensearch/common/RoundingTests.java  |  22 ---
 10 files changed, 279 insertions(+), 146 deletions(-)
 rename benchmarks/src/main/java/org/opensearch/common/{ArrayRoundingBenchmark.java => round/RoundableBenchmark.java} (89%)
 create mode 100644 libs/common/src/main/java/org/opensearch/common/round/BidirectionalLinearSearcher.java
 create mode 100644 libs/common/src/main/java/org/opensearch/common/round/BinarySearcher.java
 create mode 100644 libs/common/src/main/java/org/opensearch/common/round/Roundable.java
 create mode 100644 libs/common/src/main/java/org/opensearch/common/round/RoundableFactory.java
 create mode 100644 libs/common/src/main/java/org/opensearch/common/round/package-info.java
 create mode 100644 libs/common/src/test/java/org/opensearch/common/round/RoundableTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 202e0fa910c55..8604f0e2914ee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -130,6 +130,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add telemetry tracer/metric enable flag and integ test. ([#10395](https://github.com/opensearch-project/OpenSearch/pull/10395))
 - Add instrumentation for indexing in transport bulk action and transport shard bulk action. ([#10273](https://github.com/opensearch-project/OpenSearch/pull/10273))
 - [BUG] Disable sort optimization for HALF_FLOAT ([#10999](https://github.com/opensearch-project/OpenSearch/pull/10999))
+- Refactor common parts from the Rounding class into a separate 'round' package ([#11023](https://github.com/opensearch-project/OpenSearch/issues/11023))
 - Performance improvement for MultiTerm Queries on Keyword fields ([#7057](https://github.com/opensearch-project/OpenSearch/issues/7057))
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
 - Made leader/follower check timeout setting dynamic ([#10528](https://github.com/opensearch-project/OpenSearch/pull/10528))
diff --git a/benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java b/benchmarks/src/main/java/org/opensearch/common/round/RoundableBenchmark.java
similarity index 89%
rename from benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java
rename to benchmarks/src/main/java/org/opensearch/common/round/RoundableBenchmark.java
index 64c0a9e1d7aa6..4e07af452968b 100644
--- a/benchmarks/src/main/java/org/opensearch/common/ArrayRoundingBenchmark.java
+++ b/benchmarks/src/main/java/org/opensearch/common/round/RoundableBenchmark.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.common;
+package org.opensearch.common.round;
 
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -27,13 +27,13 @@
 @Warmup(iterations = 3, time = 1)
 @Measurement(iterations = 1, time = 1)
 @BenchmarkMode(Mode.Throughput)
-public class ArrayRoundingBenchmark {
+public class RoundableBenchmark {
 
     @Benchmark
-    public void round(Blackhole bh, Options opts) {
-        Rounding.Prepared rounding = opts.supplier.get();
+    public void floor(Blackhole bh, Options opts) {
+        Roundable roundable = opts.supplier.get();
         for (long key : opts.queries) {
-            bh.consume(rounding.round(key));
+            bh.consume(roundable.floor(key));
         }
     }
 
@@ -90,7 +90,7 @@ public static class Options {
         public String distribution;
 
         public long[] queries;
-        public Supplier supplier;
+        public Supplier supplier;
 
         @Setup
         public void setup() {
@@ -130,10 +130,10 @@ public void setup() {
 
             switch (type) {
                 case "binary":
-                    supplier = () -> new Rounding.BinarySearchArrayRounding(values, size, null);
+                    supplier = () -> new BinarySearcher(values, size);
                     break;
                 case "linear":
-                    supplier = () -> new Rounding.BidirectionalLinearSearchArrayRounding(values, size, null);
+                    supplier = () -> new BidirectionalLinearSearcher(values, size);
                     break;
                 default:
                     throw new IllegalArgumentException("invalid type: " + type);
diff --git a/libs/common/src/main/java/org/opensearch/common/round/BidirectionalLinearSearcher.java b/libs/common/src/main/java/org/opensearch/common/round/BidirectionalLinearSearcher.java
new file mode 100644
index 0000000000000..5c3dcf2bd4708
--- /dev/null
+++ b/libs/common/src/main/java/org/opensearch/common/round/BidirectionalLinearSearcher.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.round;
+
+import org.opensearch.common.annotation.InternalApi;
+
+/**
+ * It uses linear search on a sorted array of pre-computed round-down points.
+ * For small inputs (≤ 64 elements), this can be much faster than binary search as it avoids the penalty of
+ * branch mispredictions and pipeline stalls, and accesses memory sequentially.
+ *
+ * 
                              
+ * It uses "meet in the middle" linear search to avoid the worst case scenario when the desired element is present
+ * at either side of the array. This is helpful for time-series data where velocity increases over time, so more
+ * documents are likely to find a greater timestamp which is likely to be present on the right end of the array.
+ *
+ * @opensearch.internal
+ */
+@InternalApi
+class BidirectionalLinearSearcher implements Roundable {
+    private final long[] ascending;
+    private final long[] descending;
+
+    BidirectionalLinearSearcher(long[] values, int size) {
+        if (size <= 0) {
+            throw new IllegalArgumentException("at least one value must be present");
+        }
+
+        int len = (size + 1) >>> 1; // rounded-up to handle odd number of values
+        ascending = new long[len];
+        descending = new long[len];
+
+        for (int i = 0; i < len; i++) {
+            ascending[i] = values[i];
+            descending[i] = values[size - i - 1];
+        }
+    }
+
+    @Override
+    public long floor(long key) {
+        int i = 0;
+        for (; i < ascending.length; i++) {
+            if (descending[i] <= key) {
+                return descending[i];
+            }
+            if (ascending[i] > key) {
+                assert i > 0 : "key must be greater than or equal to " + ascending[0];
+                return ascending[i - 1];
+            }
+        }
+        return ascending[i - 1];
+    }
+}
diff --git a/libs/common/src/main/java/org/opensearch/common/round/BinarySearcher.java b/libs/common/src/main/java/org/opensearch/common/round/BinarySearcher.java
new file mode 100644
index 0000000000000..b9d76945115ed
--- /dev/null
+++ b/libs/common/src/main/java/org/opensearch/common/round/BinarySearcher.java
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.round;
+
+import org.opensearch.common.annotation.InternalApi;
+
+import java.util.Arrays;
+
+/**
+ * It uses binary search on a sorted array of pre-computed round-down points.
+ *
+ * @opensearch.internal
+ */
+@InternalApi
+class BinarySearcher implements Roundable {
+    private final long[] values;
+    private final int size;
+
+    BinarySearcher(long[] values, int size) {
+        if (size <= 0) {
+            throw new IllegalArgumentException("at least one value must be present");
+        }
+
+        this.values = values;
+        this.size = size;
+    }
+
+    @Override
+    public long floor(long key) {
+        int idx = Arrays.binarySearch(values, 0, size, key);
+        assert idx != -1 : "key must be greater than or equal to " + values[0];
+        if (idx < 0) {
+            idx = -2 - idx;
+        }
+        return values[idx];
+    }
+}
diff --git a/libs/common/src/main/java/org/opensearch/common/round/Roundable.java b/libs/common/src/main/java/org/opensearch/common/round/Roundable.java
new file mode 100644
index 0000000000000..ae6f9b787c1e9
--- /dev/null
+++ b/libs/common/src/main/java/org/opensearch/common/round/Roundable.java
@@ -0,0 +1,28 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.round;
+
+import org.opensearch.common.annotation.InternalApi;
+
+/**
+ * Interface to round-off values.
+ *
+ * @opensearch.internal
+ */
+@InternalApi
+@FunctionalInterface
+public interface Roundable {
+    /**
+     * Returns the greatest lower bound of the given key.
+     * In other words, it returns the largest value such that {@code value <= key}.
+     * @param key to floor
+     * @return the floored value
+     */
+    long floor(long key);
+}
diff --git a/libs/common/src/main/java/org/opensearch/common/round/RoundableFactory.java b/libs/common/src/main/java/org/opensearch/common/round/RoundableFactory.java
new file mode 100644
index 0000000000000..b7422694c3013
--- /dev/null
+++ b/libs/common/src/main/java/org/opensearch/common/round/RoundableFactory.java
@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.round;
+
+import org.opensearch.common.annotation.InternalApi;
+
+/**
+ * Factory class to create and return the fastest implementation of {@link Roundable}.
+ *
+ * @opensearch.internal
+ */
+@InternalApi
+public final class RoundableFactory {
+    /**
+     * The maximum limit up to which linear search is used, otherwise binary search is used.
+     * This is because linear search is much faster on small arrays.
+     * Benchmark results: PR #9727
+     */
+    private static final int LINEAR_SEARCH_MAX_SIZE = 64;
+
+    private RoundableFactory() {}
+
+    /**
+     * Creates and returns the fastest implementation of {@link Roundable}.
+     */
+    public static Roundable create(long[] values, int size) {
+        if (size <= LINEAR_SEARCH_MAX_SIZE) {
+            return new BidirectionalLinearSearcher(values, size);
+        } else {
+            return new BinarySearcher(values, size);
+        }
+    }
+}
diff --git a/libs/common/src/main/java/org/opensearch/common/round/package-info.java b/libs/common/src/main/java/org/opensearch/common/round/package-info.java
new file mode 100644
index 0000000000000..e79c4017de31b
--- /dev/null
+++ b/libs/common/src/main/java/org/opensearch/common/round/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Contains classes to round-off values.
+ */
+package org.opensearch.common.round;
diff --git a/libs/common/src/test/java/org/opensearch/common/round/RoundableTests.java b/libs/common/src/test/java/org/opensearch/common/round/RoundableTests.java
new file mode 100644
index 0000000000000..ae9f629c59024
--- /dev/null
+++ b/libs/common/src/test/java/org/opensearch/common/round/RoundableTests.java
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.round;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+public class RoundableTests extends OpenSearchTestCase {
+
+    public void testFloor() {
+        int size = randomIntBetween(1, 256);
+        long[] values = new long[size];
+        for (int i = 1; i < values.length; i++) {
+            values[i] = values[i - 1] + (randomNonNegativeLong() % 200) + 1;
+        }
+
+        Roundable[] impls = { new BinarySearcher(values, size), new BidirectionalLinearSearcher(values, size) };
+
+        for (int i = 0; i < 100000; i++) {
+            // Index of the expected round-down point.
+            int idx = randomIntBetween(0, size - 1);
+
+            // Value of the expected round-down point.
+            long expected = values[idx];
+
+            // Delta between the expected and the next round-down point.
+            long delta = (idx < size - 1) ? (values[idx + 1] - values[idx]) : 200;
+
+            // Adding a random delta between 0 (inclusive) and delta (exclusive) to the expected
+            // round-down point, which will still floor to the same value.
+            long key = expected + (randomNonNegativeLong() % delta);
+
+            for (Roundable roundable : impls) {
+                assertEquals(expected, roundable.floor(key));
+            }
+        }
+    }
+
+    public void testFailureCases() {
+        Throwable throwable;
+
+        throwable = assertThrows(IllegalArgumentException.class, () -> new BinarySearcher(new long[0], 0));
+        assertEquals("at least one value must be present", throwable.getMessage());
+        throwable = assertThrows(IllegalArgumentException.class, () -> new BidirectionalLinearSearcher(new long[0], 0));
+        assertEquals("at least one value must be present", throwable.getMessage());
+
+        throwable = assertThrows(AssertionError.class, () -> new BinarySearcher(new long[] { 100 }, 1).floor(50));
+        assertEquals("key must be greater than or equal to 100", throwable.getMessage());
+        throwable = assertThrows(AssertionError.class, () -> new BidirectionalLinearSearcher(new long[] { 100 }, 1).floor(50));
+        assertEquals("key must be greater than or equal to 100", throwable.getMessage());
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/Rounding.java b/server/src/main/java/org/opensearch/common/Rounding.java
index 667eb4529fe38..061934f9722f5 100644
--- a/server/src/main/java/org/opensearch/common/Rounding.java
+++ b/server/src/main/java/org/opensearch/common/Rounding.java
@@ -37,7 +37,8 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.LocalTimeOffset.Gap;
 import org.opensearch.common.LocalTimeOffset.Overlap;
-import org.opensearch.common.annotation.InternalApi;
+import org.opensearch.common.round.Roundable;
+import org.opensearch.common.round.RoundableFactory;
 import org.opensearch.common.time.DateUtils;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -59,7 +60,6 @@
 import java.time.temporal.TemporalQueries;
 import java.time.zone.ZoneOffsetTransition;
 import java.time.zone.ZoneRules;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
@@ -421,13 +421,6 @@ private abstract class PreparedRounding implements Prepared {
          */
         private static final int DEFAULT_ARRAY_ROUNDING_MAX_THRESHOLD = 128;
 
-        /**
-         * The maximum limit up to which linear search is used, otherwise binary search is used.
-         * This is because linear search is much faster on small arrays.
-         * Benchmark results: PR #9727
-         */
-        private static final int LINEAR_SEARCH_ARRAY_ROUNDING_MAX_THRESHOLD = 64;
-
         /**
          * Attempt to build a {@link Prepared} implementation that relies on pre-calcuated
          * "round down" points. If there would be more than {@code max} points then return
@@ -451,9 +444,36 @@ protected Prepared maybeUseArray(long minUtcMillis, long maxUtcMillis, int max)
                 values = ArrayUtil.grow(values, i + 1);
                 values[i++] = rounded;
             }
-            return i <= LINEAR_SEARCH_ARRAY_ROUNDING_MAX_THRESHOLD
-                ? new BidirectionalLinearSearchArrayRounding(values, i, this)
-                : new BinarySearchArrayRounding(values, i, this);
+            return new ArrayRounding(RoundableFactory.create(values, i), this);
+        }
+    }
+
+    /**
+     * ArrayRounding is an implementation of {@link Prepared} which uses
+     * pre-calculated round-down points to speed up lookups.
+     */
+    private static class ArrayRounding implements Prepared {
+        private final Roundable roundable;
+        private final Prepared delegate;
+
+        public ArrayRounding(Roundable roundable, Prepared delegate) {
+            this.roundable = roundable;
+            this.delegate = delegate;
+        }
+
+        @Override
+        public long round(long utcMillis) {
+            return roundable.floor(utcMillis);
+        }
+
+        @Override
+        public long nextRoundingValue(long utcMillis) {
+            return delegate.nextRoundingValue(utcMillis);
+        }
+
+        @Override
+        public double roundingSize(long utcMillis, DateTimeUnit timeUnit) {
+            return delegate.roundingSize(utcMillis, timeUnit);
         }
     }
 
@@ -1343,108 +1363,4 @@ public static Rounding read(StreamInput in) throws IOException {
                 throw new OpenSearchException("unknown rounding id [" + id + "]");
         }
     }
-
-    /**
-     * Implementation of {@link Prepared} using pre-calculated "round down" points.
-     *
-     * 
                              
-     * It uses binary search to find the greatest round-down point less than or equal to the given timestamp.
-     *
-     * @opensearch.internal
-     */
-    @InternalApi
-    static class BinarySearchArrayRounding implements Prepared {
-        private final long[] values;
-        private final int max;
-        private final Prepared delegate;
-
-        BinarySearchArrayRounding(long[] values, int max, Prepared delegate) {
-            assert max > 0 : "at least one round-down point must be present";
-            this.values = values;
-            this.max = max;
-            this.delegate = delegate;
-        }
-
-        @Override
-        public long round(long utcMillis) {
-            assert values[0] <= utcMillis : "utcMillis must be after " + values[0];
-            int idx = Arrays.binarySearch(values, 0, max, utcMillis);
-            assert idx != -1 : "The insertion point is before the array! This should have tripped the assertion above.";
-            assert -1 - idx <= values.length : "This insertion point is after the end of the array.";
-            if (idx < 0) {
-                idx = -2 - idx;
-            }
-            return values[idx];
-        }
-
-        @Override
-        public long nextRoundingValue(long utcMillis) {
-            return delegate.nextRoundingValue(utcMillis);
-        }
-
-        @Override
-        public double roundingSize(long utcMillis, DateTimeUnit timeUnit) {
-            return delegate.roundingSize(utcMillis, timeUnit);
-        }
-    }
-
-    /**
-     * Implementation of {@link Prepared} using pre-calculated "round down" points.
-     *
-     * 
                              
-     * It uses linear search to find the greatest round-down point less than or equal to the given timestamp.
-     * For small inputs (≤ 64 elements), this can be much faster than binary search as it avoids the penalty of
-     * branch mispredictions and pipeline stalls, and accesses memory sequentially.
-     *
-     * 
                              
-     * It uses "meet in the middle" linear search to avoid the worst case scenario when the desired element is present
-     * at either side of the array. This is helpful for time-series data where velocity increases over time, so more
-     * documents are likely to find a greater timestamp which is likely to be present on the right end of the array.
-     *
-     * @opensearch.internal
-     */
-    @InternalApi
-    static class BidirectionalLinearSearchArrayRounding implements Prepared {
-        private final long[] ascending;
-        private final long[] descending;
-        private final Prepared delegate;
-
-        BidirectionalLinearSearchArrayRounding(long[] values, int max, Prepared delegate) {
-            assert max > 0 : "at least one round-down point must be present";
-            this.delegate = delegate;
-            int len = (max + 1) >>> 1; // rounded-up to handle odd number of values
-            ascending = new long[len];
-            descending = new long[len];
-
-            for (int i = 0; i < len; i++) {
-                ascending[i] = values[i];
-                descending[i] = values[max - i - 1];
-            }
-        }
-
-        @Override
-        public long round(long utcMillis) {
-            int i = 0;
-            for (; i < ascending.length; i++) {
-                if (descending[i] <= utcMillis) {
-                    return descending[i];
-                }
-                if (ascending[i] > utcMillis) {
-                    assert i > 0 : "utcMillis must be after " + ascending[0];
-                    return ascending[i - 1];
-                }
-            }
-            return ascending[i - 1];
-        }
-
-        @Override
-        public long nextRoundingValue(long utcMillis) {
-            return delegate.nextRoundingValue(utcMillis);
-        }
-
-        @Override
-        public double roundingSize(long utcMillis, DateTimeUnit timeUnit) {
-            return delegate.roundingSize(utcMillis, timeUnit);
-        }
-    }
 }
diff --git a/server/src/test/java/org/opensearch/common/RoundingTests.java b/server/src/test/java/org/opensearch/common/RoundingTests.java
index cc71ee08abcca..9a6e6a6ac54d0 100644
--- a/server/src/test/java/org/opensearch/common/RoundingTests.java
+++ b/server/src/test/java/org/opensearch/common/RoundingTests.java
@@ -1142,28 +1142,6 @@ public void testNonMillisecondsBasedUnitCalendarRoundingSize() {
         assertThat(prepared.roundingSize(thirdQuarter, Rounding.DateTimeUnit.HOUR_OF_DAY), closeTo(2208.0, 0.000001));
     }
 
-    public void testArrayRoundingImplementations() {
-        int length = randomIntBetween(1, 256);
-        long[] values = new long[length];
-        for (int i = 1; i < values.length; i++) {
-            values[i] = values[i - 1] + (randomNonNegativeLong() % 100);
-        }
-
-        Rounding.Prepared binarySearchImpl = new Rounding.BinarySearchArrayRounding(values, length, null);
-        Rounding.Prepared linearSearchImpl = new Rounding.BidirectionalLinearSearchArrayRounding(values, length, null);
-
-        for (int i = 0; i < 100000; i++) {
-            long key = values[0] + (randomNonNegativeLong() % (100 + values[length - 1] - values[0]));
-            assertEquals(binarySearchImpl.round(key), linearSearchImpl.round(key));
-        }
-
-        AssertionError exception = expectThrows(AssertionError.class, () -> { binarySearchImpl.round(values[0] - 1); });
-        assertEquals("utcMillis must be after " + values[0], exception.getMessage());
-
-        exception = expectThrows(AssertionError.class, () -> { linearSearchImpl.round(values[0] - 1); });
-        assertEquals("utcMillis must be after " + values[0], exception.getMessage());
-    }
-
     private void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes, ZoneId tz) {
         assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz);
         long millisPerMinute = 60_000;

From 53d532996c5088e65f8652fd90548962270cca56 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Wed, 15 Nov 2023 14:04:14 -0800
Subject: [PATCH 302/388] Muting flaky tests in SegmentReplicationSuiteIT
 (#11159)

Signed-off-by: Marc Handalian 
---
 .../indices/replication/SegmentReplicationSuiteIT.java          | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java
index 800704eae7fa7..8c045c1560dd3 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.indices.replication;
 
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
@@ -15,6 +16,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Before;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9499")
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, minNumDataNodes = 2)
 public class SegmentReplicationSuiteIT extends SegmentReplicationBaseIT {
 

From 54ff35307686dbcd745e593533c61291f48a425c Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Wed, 15 Nov 2023 14:05:51 -0800
Subject: [PATCH 303/388] Fix SegmentReplicationPressureService to not schedule
 async tasks when remote failures are disabled (#10569)

* Fix SegmentReplicationPressureService to not schedule async tasks when remote failures are disabled

Today segment replication's pressure service provides a setting 'segrep.replication.time.limit' that if set to a positive value
will spawn an Async task every 30s validating if there are replicas over that limit.  If over the limit the task considers if shards should be failed remotely.
The async task is being rescheduled every 30s, even if the setting is set to 0.  This change ensures that task isn't scheduled at all and reconsiders only if
the setting is updated.

Signed-off-by: Marc Handalian 

* PR feedback.

Signed-off-by: Marc Handalian 

* Revert "PR feedback."

This reverts commit 737116104a61ec2245bda14327f50b8bf701c2a3.

Signed-off-by: Marc Handalian 

* Add better unit test to ensure task is scheduled/unscheduled.

Signed-off-by: Marc Handalian 

---------

Signed-off-by: Marc Handalian 
---
 .../SegmentReplicationPressureService.java    | 19 ++++++++--
 ...egmentReplicationPressureServiceTests.java | 37 +++++++++++++++++++
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java
index d9d480e7b2b27..ce38dd3bb236c 100644
--- a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java
@@ -98,7 +98,7 @@ public class SegmentReplicationPressureService implements Closeable {
     private final SegmentReplicationStatsTracker tracker;
     private final ShardStateAction shardStateAction;
 
-    private final AsyncFailStaleReplicaTask failStaleReplicaTask;
+    private volatile AsyncFailStaleReplicaTask failStaleReplicaTask;
 
     @Inject
     public SegmentReplicationPressureService(
@@ -202,6 +202,15 @@ public void setMaxAllowedStaleReplicas(double maxAllowedStaleReplicas) {
 
     public void setReplicationTimeLimitFailReplica(TimeValue replicationTimeLimitFailReplica) {
         this.replicationTimeLimitFailReplica = replicationTimeLimitFailReplica;
+        updateAsyncFailReplicaTask();
+    }
+
+    private synchronized void updateAsyncFailReplicaTask() {
+        try {
+            failStaleReplicaTask.close();
+        } finally {
+            failStaleReplicaTask = new AsyncFailStaleReplicaTask(this);
+        }
     }
 
     public void setReplicationTimeLimitBackpressure(TimeValue replicationTimeLimitBackpressure) {
@@ -228,13 +237,13 @@ final static class AsyncFailStaleReplicaTask extends AbstractAsyncTask {
 
         @Override
         protected boolean mustReschedule() {
-            return true;
+            return pressureService.shouldScheduleAsyncFailTask();
         }
 
         @Override
         protected void runInternal() {
             // Do not fail the replicas if time limit is set to 0 (i.e. disabled).
-            if (TimeValue.ZERO.equals(pressureService.replicationTimeLimitFailReplica) == false) {
+            if (pressureService.shouldScheduleAsyncFailTask()) {
                 final SegmentReplicationStats stats = pressureService.tracker.getStats();
 
                 // Find the shardId in node which is having stale replicas with highest current replication time.
@@ -302,4 +311,8 @@ public String toString() {
 
     }
 
+    boolean shouldScheduleAsyncFailTask() {
+        return TimeValue.ZERO.equals(replicationTimeLimitFailReplica) == false;
+    }
+
 }
diff --git a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java
index 478fdcb24f76a..a9725f638cc53 100644
--- a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java
@@ -217,6 +217,8 @@ public void testFailStaleReplicaTask() throws Exception {
             assertEquals(5, shardStats.getCheckpointsBehindCount());
 
             // call the background task
+            assertTrue(service.getFailStaleReplicaTask().mustReschedule());
+            assertTrue(service.getFailStaleReplicaTask().isScheduled());
             service.getFailStaleReplicaTask().runInternal();
 
             // verify that remote shard failed method is called which fails the replica shards falling behind.
@@ -257,6 +259,41 @@ public void testFailStaleReplicaTaskDisabled() throws Exception {
         }
     }
 
+    public void testFailStaleReplicaTaskToggleOnOff() throws Exception {
+        final Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .put(SEGMENT_REPLICATION_INDEXING_PRESSURE_ENABLED.getKey(), true)
+            .put(MAX_REPLICATION_TIME_BACKPRESSURE_SETTING.getKey(), TimeValue.timeValueMillis(10))
+            .put(MAX_REPLICATION_LIMIT_STALE_REPLICA_SETTING.getKey(), TimeValue.timeValueMillis(1))
+            .build();
+
+        try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) {
+            shards.startAll();
+            final IndexShard primaryShard = shards.getPrimary();
+            SegmentReplicationPressureService service = buildPressureService(settings, primaryShard);
+
+            // index docs in batches without refreshing
+            indexInBatches(5, shards, primaryShard);
+
+            // assert that replica shard is few checkpoints behind primary
+            Set replicationStats = primaryShard.getReplicationStatsForTrackedReplicas();
+            assertEquals(1, replicationStats.size());
+            SegmentReplicationShardStats shardStats = replicationStats.stream().findFirst().get();
+            assertEquals(5, shardStats.getCheckpointsBehindCount());
+
+            assertTrue(service.getFailStaleReplicaTask().mustReschedule());
+            assertTrue(service.getFailStaleReplicaTask().isScheduled());
+            replicateSegments(primaryShard, shards.getReplicas());
+
+            service.setReplicationTimeLimitFailReplica(TimeValue.ZERO);
+            assertFalse(service.getFailStaleReplicaTask().mustReschedule());
+            assertFalse(service.getFailStaleReplicaTask().isScheduled());
+            service.setReplicationTimeLimitFailReplica(TimeValue.timeValueMillis(1));
+            assertTrue(service.getFailStaleReplicaTask().mustReschedule());
+            assertTrue(service.getFailStaleReplicaTask().isScheduled());
+        }
+    }
+
     private int indexInBatches(int count, ReplicationGroup shards, IndexShard primaryShard) throws Exception {
         int totalDocs = 0;
         for (int i = 0; i < count; i++) {

From 5b505ec04c2684867901d1cabfed8431d7d34fe3 Mon Sep 17 00:00:00 2001
From: Jeongmin Yu 
Date: Thu, 16 Nov 2023 07:11:14 +0900
Subject: [PATCH 304/388] Treat Setting value with empty array string as empty
 array (#10625)

* Treat Setting value with empty array string as empty array

Signed-off-by: Jeongmin Yu 

* Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...)

Signed-off-by: Andriy Redko 

---------

Signed-off-by: Jeongmin Yu 
Signed-off-by: Andriy Redko 
Co-authored-by: Andriy Redko 
---
 CHANGELOG.md                                  |  1 +
 server/build.gradle                           |  1 +
 .../opensearch/common/settings/Settings.java  | 39 ++++++++++++++++++-
 .../common/settings/SettingsTests.java        | 24 ++++++++++++
 .../node/NodeRoleSettingsTests.java           | 10 +++++
 .../common/settings/loader/test-settings.yml  |  1 +
 6 files changed, 74 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8604f0e2914ee..a75463ff16a08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote cluster state] Restore global metadata from remote store when local state is lost after quorum loss ([#10404](https://github.com/opensearch-project/OpenSearch/pull/10404))
 - [AdmissionControl] Added changes for AdmissionControl Interceptor and AdmissionControlService for RateLimiting ([#9286](https://github.com/opensearch-project/OpenSearch/pull/9286))
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
+- Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
 - Add back half_float BKD based sort query optimization ([#11024](https://github.com/opensearch-project/OpenSearch/pull/11024))
 
diff --git a/server/build.gradle b/server/build.gradle
index fa8a44ef6fc94..17b8ff0469ad8 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -403,6 +403,7 @@ tasks.named("licenseHeaders").configure {
 }
 
 tasks.test {
+    environment "node.roles.test", "[]"
     if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
         jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"]
     }
diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java
index 0557884f0f8ad..9da47ff3aa700 100644
--- a/server/src/main/java/org/opensearch/common/settings/Settings.java
+++ b/server/src/main/java/org/opensearch/common/settings/Settings.java
@@ -80,6 +80,7 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
@@ -1211,8 +1212,14 @@ public boolean shouldRemoveMissingPlaceholder(String placeholderName) {
                 String value = propertyPlaceholder.replacePlaceholders(Settings.toString(entry.getValue()), placeholderResolver);
                 // if the values exists and has length, we should maintain it in the map
                 // otherwise, the replace process resolved into removing it
-                if (Strings.hasLength(value)) {
-                    entry.setValue(value);
+                if (Strings.hasLength(value) == true) {
+                    // try to parse the value as a list first
+                    final Optional> optList = tryParseableStringToList(value);
+                    if (optList.isPresent()) {
+                        entry.setValue(optList.get());
+                    } else {
+                        entry.setValue(value);
+                    }
                 } else {
                     entryItr.remove();
                 }
@@ -1248,6 +1255,34 @@ public Settings build() {
             processLegacyLists(map);
             return new Settings(map, secureSettings.get());
         }
+
+        /**
+         * Tries to parse the placeholder value as a list (fe [], ["a", "b", "c"])
+         * @param parsableString placeholder value to parse
+         * @return the {@link Optional} result of the parsing attempt
+         */
+        private static Optional> tryParseableStringToList(String parsableString) {
+            // fromXContent doesn't use named xcontent or deprecation.
+            try (
+                XContentParser xContentParser = MediaTypeRegistry.JSON.xContent()
+                    .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, parsableString)
+            ) {
+                XContentParser.Token token = xContentParser.nextToken();
+                if (token != XContentParser.Token.START_ARRAY) {
+                    return Optional.empty();
+                }
+                ArrayList list = new ArrayList<>();
+                while ((token = xContentParser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    if (token != XContentParser.Token.VALUE_STRING) {
+                        return Optional.empty();
+                    }
+                    list.add(xContentParser.text());
+                }
+                return Optional.of(list);
+            } catch (IOException e) {
+                return Optional.empty();
+            }
+        }
     }
 
     // TODO We could use an FST internally to make things even faster and more compact
diff --git a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java
index af4efabb341ee..669d40f40bb2c 100644
--- a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java
+++ b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java
@@ -50,6 +50,7 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -64,7 +65,9 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -93,6 +96,15 @@ public void testReplacePropertiesPlaceholderSystemPropertyList() {
         assertThat(settings.getAsList("setting1"), contains(hostname, hostip));
     }
 
+    public void testReplacePropertiesPlaceholderSystemPropertyEmptyList() {
+        final Settings settings = Settings.builder()
+            .put("setting1", "${HOSTNAMES}")
+            .replacePropertyPlaceholders(name -> name.equals("HOSTNAMES") ? "[]" : null)
+            .build();
+        assertThat(settings.getAsList("setting1"), empty());
+        assertThat(settings.get("setting1"), equalTo("[]"));
+    }
+
     public void testReplacePropertiesPlaceholderSystemVariablesHaveNoEffect() {
         final String value = System.getProperty("java.home");
         assertNotNull(value);
@@ -603,6 +615,18 @@ public void testSimpleYamlSettings() throws Exception {
         assertThat(settings.getAsList("test1.test3").size(), equalTo(2));
         assertThat(settings.getAsList("test1.test3").get(0), equalTo("test3-1"));
         assertThat(settings.getAsList("test1.test3").get(1), equalTo("test3-2"));
+        assertThat(settings.getAsList("test1.test4"), empty());
+    }
+
+    public void testYamlPlaceholder() throws IOException {
+        try (InputStream in = new ByteArrayInputStream("hosts: ${HOSTNAMES}".getBytes(StandardCharsets.UTF_8))) {
+            Settings settings = Settings.builder()
+                .loadFromStream("foo.yml", in, false)
+                .replacePropertyPlaceholders(name -> name.equals("HOSTNAMES") ? "[\"h1\", \"h2\"]" : null)
+                .build();
+            assertThat(settings.getAsList("hosts"), hasSize(2));
+            assertThat(settings.getAsList("hosts"), containsInAnyOrder("h1", "h2"));
+        }
     }
 
     public void testYamlLegacyList() throws IOException {
diff --git a/server/src/test/java/org/opensearch/node/NodeRoleSettingsTests.java b/server/src/test/java/org/opensearch/node/NodeRoleSettingsTests.java
index 0a3af34bc12f4..b2bb6897fe164 100644
--- a/server/src/test/java/org/opensearch/node/NodeRoleSettingsTests.java
+++ b/server/src/test/java/org/opensearch/node/NodeRoleSettingsTests.java
@@ -18,6 +18,7 @@
 import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
 
 public class NodeRoleSettingsTests extends OpenSearchTestCase {
 
@@ -72,4 +73,13 @@ public void testUnknownNodeRoleOnly() {
         assertEquals(testRole, nodeRoles.get(0).roleName());
         assertEquals(testRole, nodeRoles.get(0).roleNameAbbreviation());
     }
+
+    public void testNodeRolesFromEnvironmentVariables() {
+        Settings roleSettings = Settings.builder()
+            .put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "${node.roles.test}")
+            .replacePropertyPlaceholders()
+            .build();
+        List nodeRoles = NodeRoleSettings.NODE_ROLES_SETTING.get(roleSettings);
+        assertThat(nodeRoles, empty());
+    }
 }
diff --git a/server/src/test/resources/org/opensearch/common/settings/loader/test-settings.yml b/server/src/test/resources/org/opensearch/common/settings/loader/test-settings.yml
index b533ae036e758..1bffbc18e83e6 100644
--- a/server/src/test/resources/org/opensearch/common/settings/loader/test-settings.yml
+++ b/server/src/test/resources/org/opensearch/common/settings/loader/test-settings.yml
@@ -6,3 +6,4 @@ test1:
   test3:
     - test3-1
     - test3-2
+  test4: []

From 09bacee5fc85676e97bee6b4ad87dec35c6aa8cc Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Thu, 16 Nov 2023 10:59:19 -0500
Subject: [PATCH 305/388] [BWC and API enforcement] Decorate the existing APIs
 with proper annotations (part 3) (#11182)

Signed-off-by: Andriy Redko 
---
 .../java/org/opensearch/common/Explicit.java  |  5 +-
 .../opensearch/common/lease/Releasable.java   |  5 +-
 .../core/action/ActionResponse.java           |  2 +
 .../core/common/io/stream/Writeable.java      | 11 +++-
 .../core/common/util/ByteArray.java           |  4 +-
 .../breaker/AllCircuitBreakerStats.java       |  4 +-
 .../breaker/CircuitBreakerService.java        |  8 +--
 .../indices/breaker/CircuitBreakerStats.java  |  4 +-
 .../core/xcontent/ObjectParser.java           |  4 +-
 ...emoteStoreBackpressureAndResiliencyIT.java |  4 +-
 .../org/opensearch/action/AliasesRequest.java |  5 +-
 .../opensearch/action/OriginalIndices.java    |  4 +-
 .../ClusterAllocationExplanation.java         |  4 +-
 .../snapshots/status/SnapshotStatus.java      |  4 +-
 .../admin/indices/rollover/RolloverInfo.java  |  4 +-
 .../action/search/ParsedScrollId.java         |  5 +-
 .../action/search/SearchContextIdForNode.java |  4 +-
 .../opensearch/action/search/SearchPhase.java |  4 +-
 .../action/search/SearchPhaseContext.java     |  4 +-
 .../SearchRequestOperationsListener.java      |  4 +-
 .../action/search/SearchShardIterator.java    |  4 +-
 .../action/search/SearchShardTask.java        |  4 +-
 .../action/search/ShardSearchFailure.java     |  4 +-
 .../action/support/ActiveShardCount.java      |  4 +-
 .../action/support/RetryableAction.java       |  4 +-
 .../PendingReplicationActions.java            |  4 +-
 .../support/replication/ReplicationMode.java  |  5 +-
 .../replication/ReplicationResponse.java      | 10 +++-
 .../org/opensearch/cluster/ClusterState.java  | 10 +++-
 .../cluster/ClusterStateApplier.java          |  4 +-
 .../cluster/ClusterStateListener.java         |  5 +-
 .../cluster/ClusterStateTaskConfig.java       |  4 +-
 .../cluster/ClusterStateTaskExecutor.java     | 10 +++-
 .../cluster/ClusterStateTaskListener.java     |  4 +-
 .../cluster/ClusterStateUpdateTask.java       |  4 +-
 .../LocalNodeClusterManagerListener.java      |  5 +-
 .../cluster/LocalNodeMasterListener.java      |  5 +-
 .../org/opensearch/cluster/NamedDiff.java     |  4 +-
 .../cluster/NodeConnectionsService.java       |  4 +-
 .../opensearch/cluster/RestoreInProgress.java | 13 +++--
 .../cluster/SnapshotsInProgress.java          |  4 +-
 .../cluster/TimeoutClusterStateListener.java  |  4 +-
 .../ack/ClusterStateUpdateResponse.java       |  5 +-
 .../cluster/block/ClusterBlockException.java  |  4 +-
 .../cluster/block/ClusterBlocks.java          |  7 ++-
 .../block/IndexCreateBlockException.java      |  4 +-
 .../coordination/ClusterStatePublisher.java   |  7 ++-
 .../coordination/CoordinationMetadata.java    | 13 +++--
 .../coordination/PersistedStateStats.java     |  4 +-
 .../DecommissionAttributeMetadata.java        |  4 +-
 .../cluster/metadata/ComponentTemplate.java   |  4 +-
 .../metadata/ComposableIndexTemplate.java     |  7 ++-
 .../cluster/metadata/DiffableStringMap.java   |  4 +-
 .../cluster/metadata/IndexAbstraction.java    | 10 +++-
 .../cluster/metadata/IndexGraveyard.java      | 10 +++-
 .../cluster/metadata/IndexMetadata.java       |  9 ++-
 .../opensearch/cluster/metadata/Metadata.java | 13 +++--
 .../opensearch/cluster/metadata/Template.java |  4 +-
 .../metadata/WeightedRoutingMetadata.java     |  7 +--
 .../cluster/node/DiscoveryNode.java           | 22 --------
 .../cluster/node/DiscoveryNodeFilters.java    |  7 ++-
 .../cluster/routing/AllocationId.java         |  4 +-
 .../cluster/routing/GroupShardsIterator.java  |  4 +-
 .../cluster/routing/IndexRoutingTable.java    |  7 ++-
 .../routing/IndexShardRoutingTable.java       |  7 ++-
 .../cluster/routing/OperationRouting.java     |  4 +-
 .../cluster/routing/RecoverySource.java       |  6 +-
 .../cluster/routing/RerouteService.java       |  4 +-
 .../routing/RoutingChangesObserver.java       |  5 +-
 .../cluster/routing/RoutingNode.java          |  4 +-
 .../cluster/routing/RoutingNodes.java         | 10 +++-
 .../cluster/routing/RoutingTable.java         |  7 ++-
 .../cluster/routing/ShardIterator.java        |  4 +-
 .../routing/ShardMovementStrategy.java        |  4 +-
 .../cluster/routing/ShardRouting.java         |  4 +-
 .../cluster/routing/ShardsIterator.java       |  4 +-
 .../AllocateUnassignedDecision.java           |  4 +-
 .../allocation/AllocationDecision.java        |  4 +-
 .../routing/allocation/MoveDecision.java      |  4 +-
 .../allocation/NodeAllocationResult.java      |  7 ++-
 .../routing/allocation/RoutingAllocation.java |  7 ++-
 .../allocation/ShardAllocationDecision.java   |  4 +-
 .../allocation/command/AllocationCommand.java |  4 +-
 .../command/AllocationCommands.java           |  4 +-
 .../allocation/decider/AllocationDecider.java |  4 +-
 .../decider/AllocationDeciders.java           |  4 +-
 .../cluster/service/ClusterApplier.java       |  4 ++
 .../service/ClusterApplierService.java        |  4 +-
 .../service/ClusterManagerService.java        |  4 +-
 .../ClusterManagerThrottlingStats.java        |  4 ++
 .../cluster/service/ClusterService.java       |  4 +-
 .../cluster/service/ClusterStateStats.java    |  4 +-
 .../cluster/service/MasterService.java        |  4 +-
 .../java/org/opensearch/common/Rounding.java  | 13 +++--
 .../org/opensearch/common/cache/Cache.java    |  7 ++-
 .../opensearch/common/cache/CacheLoader.java  |  5 +-
 .../common/cache/RemovalNotification.java     |  5 +-
 .../common/cache/RemovalReason.java           |  5 +-
 .../common/compress/CompressedXContent.java   |  4 +-
 .../common/concurrent/GatedCloseable.java     |  4 +-
 .../common/io/stream/DelayableWriteable.java  |  7 ++-
 .../index/OpenSearchDirectoryReader.java      |  4 +-
 .../lucene/search/TopDocsAndMaxScore.java     |  4 +-
 .../lucene/uid/VersionsAndSeqNoResolver.java  |  4 +-
 .../opensearch/common/time/DateFormatter.java |  4 +-
 .../common/time/DateMathParser.java           |  4 +-
 .../org/opensearch/common/util/BigArrays.java |  8 ++-
 .../common/util/CancellableThreads.java       | 15 +++--
 .../org/opensearch/common/util/Countable.java |  5 +-
 .../opensearch/common/util/DoubleArray.java   |  4 +-
 .../opensearch/common/util/FloatArray.java    |  4 +-
 .../org/opensearch/common/util/IntArray.java  |  4 +-
 .../org/opensearch/common/util/LongArray.java |  4 +-
 .../opensearch/common/util/ObjectArray.java   |  4 +-
 .../util/concurrent/AsyncIOProcessor.java     |  4 +-
 .../common/util/concurrent/AtomicArray.java   |  4 +-
 .../common/util/concurrent/ThreadContext.java |  7 ++-
 .../ThreadContextStatePropagator.java         |  5 ++
 .../org/opensearch/env/NodeEnvironment.java   |  3 +
 .../gateway/WriteStateException.java          |  5 +-
 .../org/opensearch/index/IndexService.java    | 12 ++--
 .../org/opensearch/index/IndexWarmer.java     |  7 ++-
 .../index/IndexingPressureService.java        |  4 +-
 .../index/analysis/AnalysisRegistry.java      |  4 +-
 .../index/analysis/AnalyzerProvider.java      |  4 +-
 .../index/analysis/CharFilterFactory.java     |  5 +-
 .../index/analysis/IndexAnalyzers.java        |  4 +-
 .../index/analysis/TokenFilterFactory.java    |  4 +-
 .../index/analysis/TokenizerFactory.java      |  4 +-
 .../opensearch/index/cache/IndexCache.java    |  4 +-
 .../index/cache/bitset/BitsetFilterCache.java |  7 ++-
 .../cache/bitset/ShardBitsetFilterCache.java  |  4 +-
 .../index/cache/query/QueryCache.java         |  4 +-
 .../cache/request/ShardRequestCache.java      |  4 +-
 .../org/opensearch/index/engine/Engine.java   | 55 +++++++++++++------
 .../opensearch/index/engine/EngineConfig.java |  7 ++-
 .../index/engine/EngineException.java         |  4 +-
 .../index/engine/EngineFactory.java           |  5 +-
 .../index/engine/NRTReplicationEngine.java    |  4 +-
 .../index/engine/SafeCommitInfo.java          |  4 +-
 .../index/fielddata/DocValueBits.java         |  5 +-
 .../index/fielddata/IndexFieldData.java       | 13 ++++-
 .../index/fielddata/IndexFieldDataCache.java  |  7 ++-
 .../index/fielddata/LeafFieldData.java        |  4 +-
 .../index/fielddata/NumericDoubleValues.java  |  4 +-
 .../index/fielddata/ShardFieldData.java       |  4 +-
 .../fielddata/SortedNumericDoubleValues.java  |  4 +-
 .../opensearch/index/get/ShardGetService.java |  4 +-
 .../opensearch/index/mapper/ContentPath.java  |  5 +-
 .../index/mapper/DocValueFetcher.java         |  4 +-
 .../index/mapper/DocumentMapper.java          |  9 ++-
 .../index/mapper/DocumentMapperForType.java   |  3 +
 .../index/mapper/DocumentMapperParser.java    |  4 +-
 .../index/mapper/DynamicTemplate.java         |  7 ++-
 .../opensearch/index/mapper/FieldMapper.java  | 10 +++-
 .../index/mapper/FieldTypeLookup.java         |  2 +
 .../index/mapper/IdFieldMapper.java           |  4 +-
 .../index/mapper/IndexFieldMapper.java        |  4 +-
 .../index/mapper/MappedFieldType.java         |  7 ++-
 .../org/opensearch/index/mapper/Mapper.java   | 16 ++++--
 .../index/mapper/MapperParsingException.java  |  4 +-
 .../index/mapper/MapperService.java           |  5 +-
 .../org/opensearch/index/mapper/Mapping.java  |  4 +-
 .../index/mapper/MappingLookup.java           |  6 +-
 .../index/mapper/MetadataFieldMapper.java     | 10 +++-
 .../opensearch/index/mapper/ObjectMapper.java | 10 +++-
 .../index/mapper/ParametrizedFieldMapper.java | 17 ++++--
 .../opensearch/index/mapper/ParseContext.java |  7 ++-
 .../index/mapper/ParsedDocument.java          |  4 +-
 .../index/mapper/RootObjectMapper.java        |  4 +-
 .../index/mapper/RoutingFieldMapper.java      |  4 +-
 .../index/mapper/SeqNoFieldMapper.java        |  4 +-
 .../index/mapper/SourceFieldMapper.java       |  4 +-
 .../index/mapper/SourceToParse.java           |  4 +-
 .../index/mapper/TextSearchInfo.java          |  7 ++-
 .../opensearch/index/mapper/ValueFetcher.java |  4 +-
 .../index/query/InnerHitBuilder.java          |  4 ++
 .../opensearch/index/query/ParsedQuery.java   |  4 +-
 .../index/query/QueryShardContext.java        |  4 +-
 .../opensearch/index/query/Rewriteable.java   |  4 +-
 .../index/query/support/NestedScope.java      |  4 +-
 .../RemoteStoreStatsTrackerFactory.java       |  4 +-
 .../index/seqno/ReplicationTracker.java       |  7 ++-
 .../index/seqno/RetentionLeaseSyncer.java     |  2 +
 .../index/seqno/SequenceNumbers.java          |  5 +-
 .../shard/GlobalCheckpointListeners.java      |  4 +-
 .../IllegalIndexShardStateException.java      |  4 +-
 .../index/shard/IndexEventListener.java       |  4 +-
 .../opensearch/index/shard/IndexShard.java    |  7 ++-
 .../shard/IndexShardClosedException.java      |  4 +-
 .../IndexShardNotRecoveringException.java     |  4 +-
 .../shard/IndexShardNotStartedException.java  |  4 +-
 .../shard/IndexShardRecoveringException.java  |  4 +-
 .../shard/IndexShardRelocatedException.java   |  4 +-
 .../shard/IndexShardStartedException.java     |  4 +-
 .../index/shard/IndexShardState.java          |  5 +-
 .../shard/IndexingOperationListener.java      |  4 +-
 .../index/shard/PrimaryReplicaSyncer.java     |  7 ++-
 .../index/shard/ReplicationGroup.java         |  4 +-
 .../index/shard/SearchOperationListener.java  |  4 +-
 .../org/opensearch/index/shard/ShardPath.java |  4 +-
 .../index/similarity/SimilarityProvider.java  |  4 +-
 .../index/similarity/SimilarityService.java   |  4 +-
 .../snapshots/IndexShardSnapshotStatus.java   | 11 +++-
 .../RemoteStoreShardShallowCopySnapshot.java  |  4 +-
 .../stats/IndexingPressurePerShardStats.java  |  4 +-
 .../index/stats/IndexingPressureStats.java    |  4 +-
 .../stats/ShardIndexingPressureStats.java     |  4 +-
 .../store/RemoteSegmentStoreDirectory.java    |  7 ++-
 .../store/RemoteStoreFileDownloader.java      |  6 +-
 .../org/opensearch/index/store/Store.java     | 12 ++--
 .../index/store/StoreFileMetadata.java        |  4 +-
 .../index/store/lockmanager/LockInfo.java     |  6 +-
 .../lockmanager/RemoteStoreLockManager.java   |  6 +-
 .../RemoteStoreLockManagerFactory.java        |  4 +-
 .../RemoteStoreMetadataLockManager.java       |  4 +-
 .../remote/filecache/CachedIndexInput.java    |  4 +-
 .../store/remote/filecache/FileCache.java     |  4 +-
 .../metadata/RemoteSegmentMetadata.java       |  4 +-
 .../store/remote/utils/cache/CacheUsage.java  |  3 +
 .../remote/utils/cache/stats/CacheStats.java  |  4 +-
 .../index/translog/BaseTranslogReader.java    |  2 +-
 .../BufferedChecksumStreamOutput.java         |  4 +-
 .../index/translog/ChannelFactory.java        |  5 +-
 .../opensearch/index/translog/Checkpoint.java |  6 +-
 .../index/translog/TragicExceptionHolder.java |  4 +-
 .../opensearch/index/translog/Translog.java   | 21 ++++---
 .../index/translog/TranslogConfig.java        |  4 +-
 .../translog/TranslogDeletionPolicy.java      |  4 +-
 .../TranslogDeletionPolicyFactory.java        |  2 +
 .../index/translog/TranslogException.java     |  4 +-
 .../index/translog/TranslogFactory.java       |  5 +-
 .../index/translog/TranslogManager.java       |  5 +-
 .../index/translog/TranslogReader.java        |  4 +-
 .../translog/TranslogRecoveryRunner.java      |  5 +-
 .../index/translog/TranslogWriter.java        |  6 +-
 .../index/warmer/ShardIndexWarmerService.java |  4 +-
 .../opensearch/indices/IndicesQueryCache.java |  4 +-
 .../opensearch/indices/IndicesService.java    |  7 ++-
 .../opensearch/indices/NodeIndicesStats.java  |  4 +-
 .../cluster/IndicesClusterStateService.java   |  4 +-
 .../cache/IndicesFieldDataCache.java          |  7 ++-
 .../indices/mapper/MapperRegistry.java        |  4 +-
 .../recovery/PeerRecoveryTargetService.java   |  4 +-
 .../indices/recovery/RecoveryListener.java    |  2 +
 .../indices/recovery/RecoveryTarget.java      |  4 +-
 .../recovery/StartRecoveryRequest.java        |  4 +-
 .../checkpoint/PublishCheckpointAction.java   |  5 +-
 .../checkpoint/ReplicationCheckpoint.java     |  4 +-
 ...SegmentReplicationCheckpointPublisher.java |  7 ++-
 .../common/ReplicationFailedException.java    |  4 +-
 .../replication/common/ReplicationState.java  |  5 +-
 .../node/AdaptiveSelectionStats.java          |  4 +-
 .../main/java/org/opensearch/node/Node.java   | 19 +++----
 .../node/ResponseCollectorService.java        |  7 ++-
 .../opensearch/plugins/IndexStorePlugin.java  |  8 +++
 .../org/opensearch/repositories/IndexId.java  |  4 +-
 .../IndexMetaDataGenerations.java             |  4 +-
 .../opensearch/repositories/Repository.java   |  7 ++-
 .../repositories/RepositoryData.java          |  4 +-
 .../repositories/RepositoryInfo.java          |  4 +-
 .../repositories/RepositoryShardId.java       |  4 +-
 .../repositories/RepositoryStats.java         |  4 +-
 .../repositories/RepositoryStatsSnapshot.java |  4 +-
 .../repositories/ShardGenerations.java        |  7 ++-
 .../opensearch/script/AggregationScript.java  |  7 ++-
 .../org/opensearch/script/FieldScript.java    |  7 ++-
 .../org/opensearch/search/DocValueFormat.java |  4 +-
 .../org/opensearch/search/MultiValueMode.java |  4 +-
 .../org/opensearch/search/RescoreDocIds.java  |  4 +-
 .../java/org/opensearch/search/SearchHit.java |  7 ++-
 .../org/opensearch/search/SearchHits.java     |  4 +-
 .../opensearch/search/SearchPhaseResult.java  |  4 +-
 .../opensearch/search/SearchShardTarget.java  |  4 +-
 .../opensearch/search/SearchSortValues.java   |  4 +-
 .../search/SearchSortValuesAndFormats.java    |  4 +-
 .../search/aggregations/Aggregation.java      |  4 +-
 .../aggregations/AggregationBuilder.java      |  7 ++-
 .../aggregations/AggregationProcessor.java    |  4 ++
 .../search/aggregations/Aggregations.java     |  4 +-
 .../search/aggregations/Aggregator.java       |  8 ++-
 .../aggregations/AggregatorFactories.java     |  7 ++-
 .../aggregations/AggregatorFactory.java       |  4 +-
 .../BucketCollectorProcessor.java             |  4 ++
 .../aggregations/CardinalityUpperBound.java   |  4 +-
 .../aggregations/InternalAggregation.java     | 10 +++-
 .../aggregations/InternalAggregations.java    |  4 +-
 .../MultiBucketConsumerService.java           |  4 +-
 .../PipelineAggregationBuilder.java           |  4 +-
 .../SearchContextAggregations.java            |  5 +-
 .../bucket/LocalBucketCountThresholds.java    |  4 +-
 .../bucket/terms/TermsAggregator.java         |  4 +-
 .../pipeline/PipelineAggregator.java          |  7 ++-
 .../aggregations/support/AggregationInfo.java |  4 +-
 .../aggregations/support/AggregationPath.java |  4 +-
 .../support/AggregationUsageService.java      |  7 ++-
 .../aggregations/support/FieldContext.java    |  4 +-
 .../aggregations/support/ValueType.java       |  4 +-
 .../aggregations/support/ValuesSource.java    |  4 +-
 .../support/ValuesSourceConfig.java           |  4 +-
 .../support/ValuesSourceRegistry.java         | 10 +++-
 .../support/ValuesSourceType.java             |  4 +-
 .../search/builder/SearchSourceBuilder.java   | 10 +++-
 .../opensearch/search/dfs/AggregatedDfs.java  |  4 +-
 .../search/dfs/DfsSearchResult.java           |  4 +-
 .../opensearch/search/fetch/FetchPhase.java   |  4 +-
 .../search/fetch/FetchSearchResult.java       |  4 +-
 .../search/fetch/StoredFieldsContext.java     |  4 +-
 .../fetch/subphase/FetchDocValuesContext.java |  4 +-
 .../fetch/subphase/FetchFieldsContext.java    |  5 +-
 .../search/fetch/subphase/FieldAndFormat.java |  4 +-
 .../fetch/subphase/InnerHitsContext.java      |  7 ++-
 .../fetch/subphase/ScriptFieldsContext.java   |  7 ++-
 .../subphase/highlight/HighlightField.java    |  4 +-
 .../search/internal/ContextIndexSearcher.java |  4 +-
 .../internal/InternalSearchResponse.java      |  4 +-
 .../search/internal/ReaderContext.java        |  4 +-
 .../search/internal/ScrollContext.java        |  4 +-
 .../search/internal/SearchContext.java        |  4 +-
 .../search/internal/ShardSearchContextId.java |  4 +-
 .../search/internal/ShardSearchRequest.java   |  4 +-
 .../opensearch/search/lookup/DocLookup.java   |  4 +-
 .../search/lookup/LeafDocLookup.java          |  4 +-
 .../search/lookup/LeafFieldsLookup.java       |  4 +-
 .../search/lookup/LeafSearchLookup.java       |  4 +-
 .../search/lookup/SearchLookup.java           |  4 +-
 .../search/lookup/SourceLookup.java           |  4 +-
 .../opensearch/search/profile/Profilers.java  |  4 +-
 .../AggregationProfileBreakdown.java          |  4 +-
 .../aggregation/AggregationProfiler.java      |  4 +-
 .../query/InternalProfileComponent.java       |  5 +-
 .../search/profile/query/QueryProfiler.java   |  4 +-
 .../search/query/QueryCollectorContext.java   |  4 +-
 .../opensearch/search/query/QueryPhase.java   |  4 +-
 .../query/QueryPhaseExecutionException.java   |  4 +-
 .../search/query/QueryPhaseSearcher.java      |  4 +-
 .../search/query/QuerySearchResult.java       |  4 +-
 .../search/query/ReduceableSearchResult.java  |  5 +-
 .../search/rescore/RescoreContext.java        |  4 +-
 .../opensearch/search/rescore/Rescorer.java   |  4 +-
 .../search/rescore/RescorerBuilder.java       |  4 +-
 .../opensearch/search/sort/BucketedSort.java  | 16 ++++--
 .../search/sort/NestedSortBuilder.java        |  4 +-
 .../search/sort/SortAndFormats.java           |  4 +-
 .../opensearch/search/sort/SortBuilder.java   |  4 +-
 .../org/opensearch/search/sort/SortValue.java |  4 +-
 .../InternalSnapshotsInfoService.java         |  5 +-
 .../snapshots/SnapshotShardSizeInfo.java      |  4 +-
 .../org/opensearch/transport/Transport.java   |  4 ++
 .../transport/TransportException.java         |  4 +-
 .../transport/TransportRequest.java           |  4 +-
 .../transport/TransportRequestOptions.java    | 10 +++-
 .../index/IndexServiceTestUtils.java          | 20 +++++++
 353 files changed, 1404 insertions(+), 505 deletions(-)
 create mode 100644 server/src/test/java/org/opensearch/index/IndexServiceTestUtils.java

diff --git a/libs/common/src/main/java/org/opensearch/common/Explicit.java b/libs/common/src/main/java/org/opensearch/common/Explicit.java
index e060baf6f187e..da44c6fd4dcef 100644
--- a/libs/common/src/main/java/org/opensearch/common/Explicit.java
+++ b/libs/common/src/main/java/org/opensearch/common/Explicit.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.common;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Objects;
 
 /**
@@ -43,8 +45,9 @@
  * field mapping settings it is preferable to preserve an explicit
  * choice rather than a choice made only made implicitly by defaults.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Explicit {
 
     private final T value;
diff --git a/libs/common/src/main/java/org/opensearch/common/lease/Releasable.java b/libs/common/src/main/java/org/opensearch/common/lease/Releasable.java
index 30bea6185febc..dfc4fefb9ee55 100644
--- a/libs/common/src/main/java/org/opensearch/common/lease/Releasable.java
+++ b/libs/common/src/main/java/org/opensearch/common/lease/Releasable.java
@@ -32,13 +32,16 @@
 
 package org.opensearch.common.lease;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.Closeable;
 
 /**
  * Specialization of {@link AutoCloseable} for calls that might not throw a checked exception.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Releasable extends Closeable {
 
     @Override
diff --git a/libs/core/src/main/java/org/opensearch/core/action/ActionResponse.java b/libs/core/src/main/java/org/opensearch/core/action/ActionResponse.java
index 041d8b1bffb4a..7525bfb243aae 100644
--- a/libs/core/src/main/java/org/opensearch/core/action/ActionResponse.java
+++ b/libs/core/src/main/java/org/opensearch/core/action/ActionResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.action;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.transport.TransportResponse;
 
@@ -42,6 +43,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ActionResponse extends TransportResponse {
 
     public ActionResponse() {}
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java
index af9df51655414..960f4bec5eeb5 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.core.common.io.stream;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -41,8 +43,9 @@
  * across the wire" using OpenSearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by
  * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.8.0")
 public interface Writeable {
     /**
      * A WriteableRegistry registers {@link Writer} methods for writing data types over a
@@ -135,8 +138,11 @@ public static Class getCustomClassFromInstance(final Object value) {
      *     out.writeMapOfLists(someMap, StreamOutput::writeString, StreamOutput::writeString);
      * }
      * 
+     *
+     * @opensearch.api
      */
     @FunctionalInterface
+    @PublicApi(since = "2.8.0")
     interface Writer {
 
         /**
@@ -161,8 +167,11 @@ interface Writer {
      *     this.someMap = in.readMapOfLists(StreamInput::readString, StreamInput::readString);
      * }
      * 
+     *
+     * @opensearch.api
      */
     @FunctionalInterface
+    @PublicApi(since = "2.8.0")
     interface Reader {
 
         /**
diff --git a/libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java b/libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java
index e50f24417f8bc..f4d81c4ca4363 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/util/ByteArray.java
@@ -33,14 +33,16 @@
 package org.opensearch.core.common.util;
 
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.nio.ByteBuffer;
 
 /**
  * Abstraction of an array of byte values.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ByteArray extends BigArray {
 
     /**
diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java
index 3ce8b4953b9d6..992655efec8f0 100644
--- a/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.indices.breaker;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -43,8 +44,9 @@
 /**
  * Stats class encapsulating all of the different circuit breaker stats
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class AllCircuitBreakerStats implements Writeable, ToXContentFragment {
 
     /** An array of all the circuit breaker stats */
diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java
index ee9c94f432a36..dedeb0803271f 100644
--- a/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java
@@ -32,8 +32,7 @@
 
 package org.opensearch.core.indices.breaker;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
 import org.opensearch.core.common.breaker.CircuitBreaker;
 
@@ -41,11 +40,10 @@
  * Interface for Circuit Breaker services, which provide breakers to classes
  * that load field data.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class CircuitBreakerService extends AbstractLifecycleComponent {
-    private static final Logger logger = LogManager.getLogger(CircuitBreakerService.class);
-
     protected CircuitBreakerService() {}
 
     /**
diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java
index 9207d3ea77227..ee71cf8d2ac0e 100644
--- a/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.indices.breaker;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -45,8 +46,9 @@
 /**
  * Class encapsulating stats about the {@link org.opensearch.core.common.breaker.CircuitBreaker}
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CircuitBreakerStats implements Writeable, ToXContentObject {
 
     /** The name of the circuit breaker */
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/ObjectParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/ObjectParser.java
index 365b36c755dd2..04d0bce27c04f 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/ObjectParser.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/ObjectParser.java
@@ -32,6 +32,7 @@
 package org.opensearch.core.xcontent;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 
 import java.io.IOException;
@@ -83,8 +84,9 @@
  * It's highly recommended to use the high level declare methods like {@link #declareString(BiConsumer, ParseField)} instead of
  * {@link #declareField} which can be used to implement exceptional parsing operations not covered by the high level methods.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ObjectParser extends AbstractObjectParser
     implements
         BiFunction,
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
index f19c9db7874db..6b6a96dc42a84 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureAndResiliencyIT.java
@@ -14,7 +14,6 @@
 import org.opensearch.action.admin.indices.flush.FlushResponse;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.concurrent.AbstractAsyncTask;
 import org.opensearch.common.util.concurrent.UncategorizedExecutionException;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -22,6 +21,7 @@
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.IndexService;
+import org.opensearch.index.IndexServiceTestUtils;
 import org.opensearch.index.remote.RemoteSegmentTransferTracker;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
@@ -175,7 +175,7 @@ public void testAsyncTrimTaskSucceeds() {
 
         logger.info("Increasing the frequency of async trim task to ensure it runs in background while indexing");
         IndexService indexService = internalCluster().getInstance(IndicesService.class, dataNodeName).iterator().next();
-        ((AbstractAsyncTask) indexService.getTrimTranslogTask()).setInterval(TimeValue.timeValueMillis(100));
+        IndexServiceTestUtils.setTrimTranslogTaskInterval(indexService, TimeValue.timeValueMillis(100));
 
         logger.info("--> Indexing data");
         indexData(randomIntBetween(2, 5), true);
diff --git a/server/src/main/java/org/opensearch/action/AliasesRequest.java b/server/src/main/java/org/opensearch/action/AliasesRequest.java
index 3632ba2d7304f..8fe6f49c08af0 100644
--- a/server/src/main/java/org/opensearch/action/AliasesRequest.java
+++ b/server/src/main/java/org/opensearch/action/AliasesRequest.java
@@ -32,14 +32,17 @@
 
 package org.opensearch.action;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Needs to be implemented by all {@link org.opensearch.action.ActionRequest} subclasses that relate to
  * one or more indices and one or more aliases. Meant to be used for aliases management requests (e.g. add/remove alias,
  * get aliases) that hold aliases and indices in separate fields.
  * Allows to retrieve which indices and aliases the action relates to.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface AliasesRequest extends IndicesRequest.Replaceable {
 
     /**
diff --git a/server/src/main/java/org/opensearch/action/OriginalIndices.java b/server/src/main/java/org/opensearch/action/OriginalIndices.java
index 1e24c64bc60fc..1c26bf9d416f5 100644
--- a/server/src/main/java/org/opensearch/action/OriginalIndices.java
+++ b/server/src/main/java/org/opensearch/action/OriginalIndices.java
@@ -33,6 +33,7 @@
 package org.opensearch.action;
 
 import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -42,8 +43,9 @@
 /**
  * Used to keep track of original indices within internal (e.g. shard level) requests
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class OriginalIndices implements IndicesRequest {
 
     // constant to use when original indices are not applicable and will not be serialized across the wire
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index f4705a21f5014..3c8f07613561d 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -40,6 +40,7 @@
 import org.opensearch.cluster.routing.allocation.AllocationDecision;
 import org.opensearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -58,8 +59,9 @@
  * or if it is not unassigned, then which nodes it could possibly be relocated to.
  * It is an immutable class.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ClusterAllocationExplanation implements ToXContentObject, Writeable {
 
     private final ShardRouting shardRouting;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
index 0b101070db478..e0f380b3ebbe6 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.SnapshotsInProgress;
 import org.opensearch.cluster.SnapshotsInProgress.State;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -68,8 +69,9 @@
 /**
  * Status of a snapshot
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SnapshotStatus implements ToXContentObject, Writeable {
 
     private final Snapshot snapshot;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java
index 475e44e1820d5..75c68350e2204 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverInfo.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -52,8 +53,9 @@
 /**
  * Class for holding Rollover related information within an index
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RolloverInfo extends AbstractDiffable implements Writeable, ToXContentFragment {
 
     public static final ParseField CONDITION_FIELD = new ParseField("met_conditions");
diff --git a/server/src/main/java/org/opensearch/action/search/ParsedScrollId.java b/server/src/main/java/org/opensearch/action/search/ParsedScrollId.java
index 70be9a0b19e08..b723b97b5c413 100644
--- a/server/src/main/java/org/opensearch/action/search/ParsedScrollId.java
+++ b/server/src/main/java/org/opensearch/action/search/ParsedScrollId.java
@@ -32,13 +32,16 @@
 
 package org.opensearch.action.search;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Arrays;
 
 /**
  * Search scroll id that has been parsed
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ParsedScrollId {
 
     public static final String QUERY_THEN_FETCH_TYPE = "queryThenFetch";
diff --git a/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java
index 59c0e54bb6cbc..7c702c16030b7 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchContextIdForNode.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.search;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -43,8 +44,9 @@
 /**
  * Id for a search context per node.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class SearchContextIdForNode implements Writeable {
     private final String node;
     private final ShardSearchContextId searchContextId;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhase.java b/server/src/main/java/org/opensearch/action/search/SearchPhase.java
index 1c7b3c1f1563c..0890e9f5de8d4 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchPhase.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchPhase.java
@@ -32,6 +32,7 @@
 package org.opensearch.action.search;
 
 import org.opensearch.common.CheckedRunnable;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -40,8 +41,9 @@
 /**
  * Base class for all individual search phases like collecting distributed frequencies, fetching documents, querying shards.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class SearchPhase implements CheckedRunnable {
     private final String name;
     private long startTimeInNanos;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java
index 45d39a6f85ea2..0fa8569413eaf 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseContext.java
@@ -34,6 +34,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.OriginalIndices;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.util.concurrent.AtomicArray;
 import org.opensearch.search.SearchPhaseResult;
@@ -48,8 +49,9 @@
 /**
  * This class provide contextual state and access to resources across multiple search phases.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface SearchPhaseContext extends Executor {
     // TODO maybe we can make this concrete later - for now we just implement this in the base class for all initial phases
 
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java b/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java
index 89d725b56bded..ecf48b4a9f633 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java
@@ -10,14 +10,16 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.List;
 
 /**
  * A listener for search, fetch and context events at the coordinator node level
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface SearchRequestOperationsListener {
 
     void onPhaseStart(SearchPhaseContext context);
diff --git a/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java b/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java
index fbd85a3fc0b8f..33d9bf70021a0 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchShardIterator.java
@@ -36,6 +36,7 @@
 import org.opensearch.cluster.routing.PlainShardIterator;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.Countable;
 import org.opensearch.common.util.PlainIterator;
@@ -54,8 +55,9 @@
  * the cluster alias.
  * @see OriginalIndices
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class SearchShardIterator implements Comparable, Countable {
 
     private final OriginalIndices originalIndices;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchShardTask.java b/server/src/main/java/org/opensearch/action/search/SearchShardTask.java
index 54faaf363cb70..dfecf4f462c4d 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchShardTask.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchShardTask.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.search;
 
 import org.opensearch.common.MemoizedSupplier;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.tasks.TaskId;
 import org.opensearch.search.fetch.ShardFetchSearchRequest;
 import org.opensearch.search.internal.ShardSearchRequest;
@@ -46,8 +47,9 @@
  * Task storing information about a currently running search shard request.
  * See {@link ShardSearchRequest}, {@link ShardFetchSearchRequest}, ...
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class SearchShardTask extends CancellableTask implements SearchBackpressureTask {
     // generating metadata in a lazy way since source can be quite big
     private final MemoizedSupplier metadataSupplier;
diff --git a/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java
index 4d3f9b71a919c..ef490844db692 100644
--- a/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java
+++ b/server/src/main/java/org/opensearch/action/search/ShardSearchFailure.java
@@ -37,6 +37,7 @@
 import org.opensearch.action.OriginalIndices;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -56,8 +57,9 @@
 /**
  * Represents a failure to search on a specific shard.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ShardSearchFailure extends ShardOperationFailedException {
 
     private static final String REASON_FIELD = "reason";
diff --git a/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java b/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java
index 15275ba48fc6e..e91342a7ce4b8 100644
--- a/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java
+++ b/server/src/main/java/org/opensearch/action/support/ActiveShardCount.java
@@ -36,6 +36,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -48,8 +49,9 @@
  * A class whose instances represent a value for counting the number
  * of active shard copies for a given shard in an index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ActiveShardCount implements Writeable {
 
     private static final int ACTIVE_SHARD_COUNT_DEFAULT = -2;
diff --git a/server/src/main/java/org/opensearch/action/support/RetryableAction.java b/server/src/main/java/org/opensearch/action/support/RetryableAction.java
index fc2ee277b538a..d555a876f20fb 100644
--- a/server/src/main/java/org/opensearch/action/support/RetryableAction.java
+++ b/server/src/main/java/org/opensearch/action/support/RetryableAction.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.action.ActionRunnable;
 import org.opensearch.action.bulk.BackoffPolicy;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
@@ -52,8 +53,9 @@
  * default. The action will be retried with exponentially increasing delay periods until the timeout period
  * has been reached.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class RetryableAction {
 
     private final Logger logger;
diff --git a/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java b/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java
index c52bc6c2c799d..bb6bfc6fd4773 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/PendingReplicationActions.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.support.replication;
 
 import org.opensearch.action.support.RetryableAction;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.util.concurrent.ConcurrentCollections;
 import org.opensearch.core.index.shard.ShardId;
@@ -51,8 +52,9 @@
 /**
  * Pending Replication Actions
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class PendingReplicationActions implements Consumer, Releasable {
 
     private final Map>> onGoingReplicationActions = ConcurrentCollections.newConcurrentMap();
diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationMode.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationMode.java
index f9b85cc4bd7aa..dd3a38c1398ab 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationMode.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationMode.java
@@ -8,11 +8,14 @@
 
 package org.opensearch.action.support.replication;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * The type of replication used for inter-node replication.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public enum ReplicationMode {
     /**
      * In this mode, a {@code TransportReplicationAction} is fanned out to underlying concerned shard and is replicated logically.
diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java
index 825405cb68549..67480976c500f 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationResponse.java
@@ -36,6 +36,7 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.action.ShardOperationFailedException;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -57,8 +58,9 @@
 /**
  * Base class for write action responses.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ReplicationResponse extends ActionResponse {
 
     public static final ReplicationResponse.ShardInfo.Failure[] EMPTY = new ReplicationResponse.ShardInfo.Failure[0];
@@ -88,8 +90,9 @@ public void setShardInfo(ShardInfo shardInfo) {
     /**
      * Holds shard information
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ShardInfo implements Writeable, ToXContentObject {
 
         private static final String TOTAL = "total";
@@ -235,8 +238,9 @@ public String toString() {
         /**
          * Holds failure information
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public static class Failure extends ShardOperationFailedException implements ToXContentObject {
 
             private static final String _INDEX = "_index";
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java
index 1b87a60c2ccf5..9e63f961d241d 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java
@@ -49,6 +49,7 @@
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
@@ -97,8 +98,9 @@
  * throws the {@link IncompatibleClusterStateVersionException}, which causes the publishing mechanism to send
  * a full version of the cluster state to the node on which this exception was thrown.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterState implements ToXContentFragment, Diffable {
 
     public static final ClusterState EMPTY_STATE = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
@@ -139,8 +141,9 @@ static  boolean shouldSerializ
     /**
      * Custom cluster state.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public interface Custom extends NamedDiffable, ToXContentFragment, FeatureAware {
 
         /**
@@ -596,8 +599,9 @@ public static Builder builder(ClusterState state) {
     /**
      * Builder for cluster state.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
 
         private final ClusterName clusterName;
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterStateApplier.java b/server/src/main/java/org/opensearch/cluster/ClusterStateApplier.java
index 140e6426bb801..d7702729ae884 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterStateApplier.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterStateApplier.java
@@ -33,13 +33,15 @@
 package org.opensearch.cluster;
 
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.annotation.PublicApi;
 
 /**
  * A component that is in charge of applying an incoming cluster state to the node internal data structures.
  * The single apply method is called before the cluster state becomes visible via {@link ClusterService#state()}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ClusterStateApplier {
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterStateListener.java b/server/src/main/java/org/opensearch/cluster/ClusterStateListener.java
index 01a8e51a3d13e..57fcfcad099ed 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterStateListener.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterStateListener.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.cluster;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * A listener to be notified when a cluster state changes.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ClusterStateListener {
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskConfig.java b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskConfig.java
index 9a4b708548a7d..149d93a158007 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskConfig.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskConfig.java
@@ -33,13 +33,15 @@
 
 import org.opensearch.common.Nullable;
 import org.opensearch.common.Priority;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
  * Cluster state update task configuration for timeout and priority
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ClusterStateTaskConfig {
     /**
      * The timeout for this cluster state update task configuration. If
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java
index bf8494cc36857..115816798959e 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskExecutor.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.cluster.service.ClusterManagerTaskThrottler;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.IdentityHashMap;
 import java.util.List;
@@ -41,8 +42,9 @@
 /**
  * Interface that updates the cluster state based on the task
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ClusterStateTaskExecutor {
     /**
      * Update the cluster state based on the current state and the given tasks. Return the *same instance* if no state
@@ -103,8 +105,9 @@ default ClusterManagerTaskThrottler.ThrottlingKey getClusterManagerThrottlingKey
      * Represents the result of a batched execution of cluster state update tasks
      * @param  the type of the cluster state update task
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     class ClusterTasksResult {
         @Nullable
         public final ClusterState resultingState;
@@ -127,8 +130,9 @@ public static  Builder builder() {
         /**
          * Builder for cluster state task.
          *
-         * @opensearch.internal
+         * @opensearch.api
          */
+        @PublicApi(since = "1.0.0")
         public static class Builder {
             private final Map executionResults = new IdentityHashMap<>();
 
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskListener.java b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskListener.java
index d6c4abfad7b8d..0cb24bd3f3eab 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterStateTaskListener.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterStateTaskListener.java
@@ -32,14 +32,16 @@
 package org.opensearch.cluster;
 
 import org.opensearch.cluster.service.ClusterManagerService;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.List;
 
 /**
  * Interface to implement a cluster state change listener
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ClusterStateTaskListener {
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterStateUpdateTask.java b/server/src/main/java/org/opensearch/cluster/ClusterStateUpdateTask.java
index 9225914a931b2..ae6626dd4785d 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterStateUpdateTask.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterStateUpdateTask.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.common.Nullable;
 import org.opensearch.common.Priority;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 import java.util.List;
@@ -41,8 +42,9 @@
 /**
  * A task that can update the cluster state.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ClusterStateUpdateTask
     implements
         ClusterStateTaskConfig,
diff --git a/server/src/main/java/org/opensearch/cluster/LocalNodeClusterManagerListener.java b/server/src/main/java/org/opensearch/cluster/LocalNodeClusterManagerListener.java
index c07dcc5daaee6..0b2ed7ef66d43 100644
--- a/server/src/main/java/org/opensearch/cluster/LocalNodeClusterManagerListener.java
+++ b/server/src/main/java/org/opensearch/cluster/LocalNodeClusterManagerListener.java
@@ -31,12 +31,15 @@
 
 package org.opensearch.cluster;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Enables listening to cluster-manager changes events of the local node (when the local node becomes the cluster-manager, and when the local
  * node cease being a cluster-manager).
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface LocalNodeClusterManagerListener extends ClusterStateListener {
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/LocalNodeMasterListener.java b/server/src/main/java/org/opensearch/cluster/LocalNodeMasterListener.java
index 31c0b294b8004..2487aaf0d7c51 100644
--- a/server/src/main/java/org/opensearch/cluster/LocalNodeMasterListener.java
+++ b/server/src/main/java/org/opensearch/cluster/LocalNodeMasterListener.java
@@ -31,14 +31,17 @@
 
 package org.opensearch.cluster;
 
+import org.opensearch.common.annotation.DeprecatedApi;
+
 /**
  * Enables listening to cluster-manager changes events of the local node (when the local node becomes the cluster-manager, and when the local
  * node cease being a cluster-manager).
  *
- * @opensearch.internal
+ * @opensearch.api
  * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link LocalNodeClusterManagerListener}
  */
 @Deprecated
+@DeprecatedApi(since = "2.2.0")
 public interface LocalNodeMasterListener extends LocalNodeClusterManagerListener {
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/NamedDiff.java b/server/src/main/java/org/opensearch/cluster/NamedDiff.java
index ce971aa723394..e994cfb224386 100644
--- a/server/src/main/java/org/opensearch/cluster/NamedDiff.java
+++ b/server/src/main/java/org/opensearch/cluster/NamedDiff.java
@@ -33,13 +33,15 @@
 package org.opensearch.cluster;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.NamedWriteable;
 
 /**
  * Diff that also support NamedWriteable interface
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface NamedDiff> extends Diff, NamedWriteable {
     /**
      * The minimal version of the recipient this custom object can be sent to
diff --git a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
index 5b7bb19a935b1..1c12c260b3929 100644
--- a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
@@ -43,6 +43,7 @@
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterApplier;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
 import org.opensearch.common.settings.Setting;
@@ -81,8 +82,9 @@
  * This component does not block on disconnections at all, because a disconnection might need to wait for an ongoing (background) connection
  * attempt to complete first.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodeConnectionsService extends AbstractLifecycleComponent {
     private static final Logger logger = LogManager.getLogger(NodeConnectionsService.class);
 
diff --git a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java
index 3e0c78099e6b4..769f97373f7b7 100644
--- a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java
+++ b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterState.Custom;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -54,8 +55,9 @@
 /**
  * Meta data about restore processes that are currently executing
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class RestoreInProgress extends AbstractNamedDiffable implements Custom, Iterable {
 
     /**
@@ -139,8 +141,9 @@ public RestoreInProgress build() {
     /**
      * Restore metadata
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Entry {
         private final String uuid;
         private final State state;
@@ -238,8 +241,9 @@ public int hashCode() {
     /**
      * Represents status of a restored shard
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class ShardRestoreStatus implements Writeable {
         private State state;
         private String nodeId;
@@ -363,8 +367,9 @@ public int hashCode() {
     /**
      * Shard restore process state
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum State {
         /**
          * Initializing state
diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
index 952591f346536..3de23d2490c63 100644
--- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
+++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterState.Custom;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -928,8 +929,9 @@ public String toString() {
     /**
      * State of the snapshots.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum State {
         INIT((byte) 0, false),
         STARTED((byte) 1, false),
diff --git a/server/src/main/java/org/opensearch/cluster/TimeoutClusterStateListener.java b/server/src/main/java/org/opensearch/cluster/TimeoutClusterStateListener.java
index eb31fa2b7e69d..f0fa5af64d157 100644
--- a/server/src/main/java/org/opensearch/cluster/TimeoutClusterStateListener.java
+++ b/server/src/main/java/org/opensearch/cluster/TimeoutClusterStateListener.java
@@ -32,13 +32,15 @@
 
 package org.opensearch.cluster;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 
 /**
  * An exception to cluster state listener that allows for timeouts and for post added notifications.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TimeoutClusterStateListener extends ClusterStateListener {
 
     void postAdded();
diff --git a/server/src/main/java/org/opensearch/cluster/ack/ClusterStateUpdateResponse.java b/server/src/main/java/org/opensearch/cluster/ack/ClusterStateUpdateResponse.java
index db26496c6f263..837bf8af449ad 100644
--- a/server/src/main/java/org/opensearch/cluster/ack/ClusterStateUpdateResponse.java
+++ b/server/src/main/java/org/opensearch/cluster/ack/ClusterStateUpdateResponse.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.cluster.ack;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Base response returned after a cluster state update
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterStateUpdateResponse {
 
     private final boolean acknowledged;
diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java
index 4673f075e8439..ea6c0c69c523d 100644
--- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java
+++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockException.java
@@ -33,6 +33,7 @@
 package org.opensearch.cluster.block;
 
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.rest.RestStatus;
@@ -49,8 +50,9 @@
 /**
  * Internal exception on obtaining a cluster block
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterBlockException extends OpenSearchException {
     private final Set blocks;
 
diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java
index bcb4f7f71268e..304136166d515 100644
--- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java
+++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java
@@ -37,6 +37,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MetadataIndexStateService;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -59,8 +60,9 @@
 /**
  * Represents current cluster level blocks to block dirty operations done against the cluster.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ClusterBlocks extends AbstractDiffable {
     public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(emptySet(), Map.of());
 
@@ -355,8 +357,9 @@ public static Builder builder() {
     /**
      * Builder for cluster blocks.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
 
         private final Set global = new HashSet<>();
diff --git a/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java b/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java
index 729d76c72e99e..3e9fa3e23a12f 100644
--- a/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java
+++ b/server/src/main/java/org/opensearch/cluster/block/IndexCreateBlockException.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.cluster.block;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -16,8 +17,9 @@
 /**
  * Internal exception on obtaining an index create block
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexCreateBlockException extends ClusterBlockException {
 
     public IndexCreateBlockException(Set globalLevelBlocks) {
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java b/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java
index 3a506397bcac8..0ef8262a216ee 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/ClusterStatePublisher.java
@@ -34,14 +34,16 @@
 import org.opensearch.cluster.ClusterChangedEvent;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
 
 /**
  * Publishes the cluster state
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ClusterStatePublisher {
     /**
      * Publish all the changes to the cluster from the cluster-manager (can be called just by the cluster-manager). The publish
@@ -59,8 +61,9 @@ public interface ClusterStatePublisher {
     /**
      * An acknowledgement listener.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     interface AckListener {
         /**
          * Should be called when the cluster coordination layer has committed the cluster state (i.e. even if this publication fails,
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java
index 869c86ae75b90..53398d6f3f98f 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java
@@ -32,6 +32,7 @@
 package org.opensearch.cluster.coordination;
 
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -55,8 +56,9 @@
 /**
  * Metadata for cluster coordination
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class CoordinationMetadata implements Writeable, ToXContentFragment {
 
     public static final CoordinationMetadata EMPTY_METADATA = builder().build();
@@ -214,8 +216,9 @@ public String toString() {
     /**
      * Builder for coordination metadata.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
         private long term = 0;
         private VotingConfiguration lastCommittedConfiguration = VotingConfiguration.EMPTY_CONFIG;
@@ -266,8 +269,9 @@ public CoordinationMetadata build() {
     /**
      * Excluded nodes from voting config.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class VotingConfigExclusion implements Writeable, ToXContentFragment {
         public static final String MISSING_VALUE_MARKER = "_absent_";
         private final String nodeId;
@@ -362,8 +366,9 @@ public String toString() {
     /**
      * A collection of persistent node ids, denoting the voting configuration for cluster state changes.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class VotingConfiguration implements Writeable, ToXContentFragment {
 
         public static final VotingConfiguration EMPTY_CONFIG = new VotingConfiguration(Collections.emptySet());
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java b/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
index 4d466c4b3ad73..0b7ed4fee5775 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/PersistedStateStats.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.cluster.coordination;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -22,8 +23,9 @@
 /**
  * Persisted cluster state related stats.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.12.0")
 public class PersistedStateStats implements Writeable, ToXContentObject {
     private final String statsName;
     private AtomicLong totalTimeInMillis = new AtomicLong(0);
diff --git a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java
index 3df807a4f94d3..254d70b0422d7 100644
--- a/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/decommission/DecommissionAttributeMetadata.java
@@ -14,6 +14,7 @@
 import org.opensearch.cluster.NamedDiff;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.Metadata.Custom;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -30,8 +31,9 @@
 /**
  * Contains metadata about decommission attribute
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.4.0")
 public class DecommissionAttributeMetadata extends AbstractNamedDiffable implements Custom {
 
     public static final String TYPE = "decommissionedAttribute";
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java
index cab1b75afe3e1..abc3712ee07e3 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/ComponentTemplate.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.cluster.Diff;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -55,8 +56,9 @@
  * contains a field "foo", it's expected to contain all the necessary settings/mappings/etc for the
  * "foo" field. These component templates make up the individual pieces composing an index template.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ComponentTemplate extends AbstractDiffable implements ToXContentObject {
     private static final ParseField TEMPLATE = new ParseField("template");
     private static final ParseField VERSION = new ParseField("version");
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java
index 5f1291f6c6d82..e7f1b97f28842 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/ComposableIndexTemplate.java
@@ -36,6 +36,7 @@
 import org.opensearch.cluster.Diff;
 import org.opensearch.cluster.metadata.DataStream.TimestampField;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -63,8 +64,9 @@
  * ids corresponding to component templates that should be composed in order when creating a new
  * index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ComposableIndexTemplate extends AbstractDiffable implements ToXContentObject {
     private static final ParseField INDEX_PATTERNS = new ParseField("index_patterns");
     private static final ParseField TEMPLATE = new ParseField("template");
@@ -284,8 +286,9 @@ public String toString() {
     /**
      * Template for data stream.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class DataStreamTemplate implements Writeable, ToXContentObject {
 
         private static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java
index 8209c7bb56ad8..a8102182576ff 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.cluster.Diff;
 import org.opensearch.cluster.Diffable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -50,8 +51,9 @@
  * This is a {@code Map} that implements AbstractDiffable so it
  * can be used for cluster state purposes
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class DiffableStringMap extends AbstractMap implements Diffable {
 
     public static final DiffableStringMap EMPTY = new DiffableStringMap(Collections.emptyMap());
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstraction.java
index 0c316373e484f..0f3ee894a7f63 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstraction.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstraction.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.common.Nullable;
 import org.opensearch.common.SetOnce;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.core.common.Strings;
 
@@ -52,8 +53,9 @@
  * An index abstraction has a unique name and encapsulates all the  {@link IndexMetadata} instances it is pointing to.
  * Also depending on type it may refer to a single or many concrete indices and may or may not have a write index.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface IndexAbstraction {
 
     /**
@@ -102,7 +104,10 @@ public interface IndexAbstraction {
 
     /**
      * An index abstraction type.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     enum Type {
 
         /**
@@ -335,8 +340,9 @@ private boolean isNonEmpty(List idxMetas) {
     /**
      * A data stream.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     class DataStream implements IndexAbstraction {
 
         private final org.opensearch.cluster.metadata.DataStream dataStream;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java
index 85a203e5e059a..b4f8b6b188531 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexGraveyard.java
@@ -35,6 +35,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.Diff;
 import org.opensearch.cluster.NamedDiff;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
@@ -68,8 +69,9 @@
  * tombstones remain in the cluster state for a fixed period of time, after which
  * they are purged.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class IndexGraveyard implements Metadata.Custom {
 
     /**
@@ -191,8 +193,9 @@ public static IndexGraveyard.Builder builder(final IndexGraveyard graveyard) {
     /**
      * A class to build an IndexGraveyard.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static final class Builder {
         private List tombstones;
         private int numPurged = -1;
@@ -367,8 +370,9 @@ public String getWriteableName() {
     /**
      * An individual tombstone entry for representing a deleted index.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static final class Tombstone implements ToXContentObject, Writeable {
 
         private static final String INDEX_KEY = "index";
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
index 2e1421c278879..03784df509ed6 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -97,8 +97,9 @@
 /**
  * Index metadata information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class IndexMetadata implements Diffable, ToXContentFragment {
 
     public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(
@@ -160,8 +161,9 @@ public class IndexMetadata implements Diffable, ToXContentFragmen
     /**
      * The state of the index.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum State {
         OPEN((byte) 0),
         CLOSE((byte) 1);
@@ -1211,8 +1213,9 @@ public static Builder builder(IndexMetadata indexMetadata) {
     /**
      * Builder of index metadata.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
 
         private String index;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
index 70c1d059a1b9e..1871ed24973c2 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
@@ -49,6 +49,7 @@
 import org.opensearch.cluster.decommission.DecommissionAttributeMetadata;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
@@ -98,8 +99,9 @@
 /**
  * Metadata information
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Metadata implements Iterable, Diffable, ToXContentFragment {
 
     private static final Logger logger = LogManager.getLogger(Metadata.class);
@@ -127,8 +129,9 @@ public boolean isSegmentReplicationEnabled(String indexName) {
     /**
      * Context of the XContent.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public enum XContentContext {
         /* Custom metadata should be returns as part of API call */
         API,
@@ -166,8 +169,9 @@ public enum XContentContext {
     /**
      * Custom metadata.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public interface Custom extends NamedDiffable, ToXContentFragment, ClusterState.FeatureAware {
 
         EnumSet context();
@@ -1129,8 +1133,9 @@ public static Builder builder(Metadata metadata) {
     /**
      * Builder of metadata.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static class Builder {
 
         private String clusterUUID;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Template.java b/server/src/main/java/org/opensearch/cluster/metadata/Template.java
index 8e367c71ed166..bd110c6af8975 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/Template.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/Template.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.cluster.AbstractDiffable;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.compress.CompressedXContent;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentHelper;
@@ -58,8 +59,9 @@
  * it is entirely independent from an index. It's a building block forming part of a regular index
  * template and a {@link ComponentTemplate}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Template extends AbstractDiffable