Updated changes to security

.ci/bwcVersions

@@ -48,4 +48,5 @@ BWC_VERSION:
   - "2.1.0"
   - "2.1.1"
   - "2.2.0"
+  - "2.2.1"
   - "2.3.0"

DEVELOPER_GUIDE.md

@@ -4,14 +4,16 @@
     - [Install Prerequisites](#install-prerequisites)
       - [JDK 11](#jdk-11)
       - [JDK 14](#jdk-14)
-      - [Runtime JDK](#runtime-jdk)
+      - [JDK 17](#jdk-17)
+      - [Custom Runtime JDK](#custom-runtime-jdk)
       - [Windows](#windows)
       - [Docker](#docker)
     - [Build](#build)
     - [Run Tests](#run-tests)
     - [Run OpenSearch](#run-opensearch)
   - [Use an Editor](#use-an-editor)
     - [IntelliJ IDEA](#intellij-idea)
+      - [Remote development using JetBrains Gateway](#remote-development-using-jetbrains-gateway)
     - [Visual Studio Code](#visual-studio-code)
     - [Eclipse](#eclipse)
   - [Project Layout](#project-layout)
@@ -35,6 +37,7 @@
       - [testImplementation](#testimplementation)
     - [Gradle Plugins](#gradle-plugins)
       - [Distribution Download Plugin](#distribution-download-plugin)
+    - [Creating fat-JAR of a Module](#creating-fat-jar-of-a-module)
   - [Misc](#misc)
     - [git-secrets](#git-secrets)
       - [Installation](#installation)
@@ -49,7 +52,7 @@
   - [Submitting Changes](#submitting-changes)
   - [Backports](#backports)
   - [LineLint](#linelint)
-  - [Lucene Snapshots](#lucene-snapshots)
+- [Lucene Snapshots](#lucene-snapshots)
 
 # Developer Guide
 
@@ -374,6 +377,42 @@ The Distribution Download plugin downloads the latest version of OpenSearch by d
 ./gradlew integTest -PcustomDistributionUrl="https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1127/linux/x64/dist/opensearch-1.2.0-linux-x64.tar.gz"
 ```
 
+### Creating fat-JAR of a Module
+
+A fat-JAR (or an uber-JAR) is the JAR, which contains classes from all the libraries, on which your project depends and, of course, the classes of current project.
+
+There might be cases where a developer would like to add some custom logic to the code of a module (or multiple modules) and generate a fat-JAR that can be directly used by the dependency management tool. For example, in [#3665](https://github.com/opensearch-project/OpenSearch/pull/3665) a developer wanted to provide a tentative patch as a fat-JAR to a consumer for changes made in the high level REST client. 
+
+Use [Gradle Shadow plugin](https://imperceptiblethoughts.com/shadow/).
+Add the following to the `build.gradle` file of the module for which you want to create the fat-JAR, e.g. `client/rest-high-level/build.gradle`:
+
+```
+apply plugin: 'com.github.johnrengelman.shadow'
+```
+
+Run the `shadowJar` command using:
+```
+./gradlew :client:rest-high-level:shadowJar
+```
+
+This will generate a fat-JAR in the `build/distributions` folder of the module, e.g. .`/client/rest-high-level/build/distributions/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar`.
+
+You can further customize your fat-JAR by customising the plugin, More information about shadow plugin can be found [here](https://imperceptiblethoughts.com/shadow/).
+
+To use the generated JAR, install the JAR locally, e.g. 
+```
+mvn install:install-file -Dfile=src/main/resources/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar -DgroupId=org.opensearch.client -DartifactId=opensearch-rest-high-level-client -Dversion=1.4.0-SNAPSHOT -Dpackaging=jar -DgeneratePom=true
+```
+
+Refer the installed JAR as any other maven artifact, e.g.
+
+```
+<dependency>
+    <groupId>org.opensearch.client</groupId>
+    <artifactId>opensearch-rest-high-level-client</artifactId>
+    <version>1.4.0-SNAPSHOT</version>
+</dependency>
+```
 
 ## Misc
 

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.9.0" apply false
+  id "com.diffplug.spotless" version "6.9.1" apply false
   id "org.gradle.test-retry" version "1.4.0" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.3.0
+lucene            = 9.4.0-snapshot-ddf0d0a
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.4+8

client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java

@@ -112,10 +112,6 @@
 import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.filter.ParsedFilter;
 import org.opensearch.search.aggregations.bucket.filter.ParsedFilters;
-import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
-import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid;
 import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.global.ParsedGlobal;
 import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
@@ -161,7 +157,6 @@
 import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks;
 import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles;
@@ -173,7 +168,6 @@
 import org.opensearch.search.aggregations.metrics.ParsedAvg;
 import org.opensearch.search.aggregations.metrics.ParsedCardinality;
 import org.opensearch.search.aggregations.metrics.ParsedExtendedStats;
-import org.opensearch.search.aggregations.metrics.ParsedGeoBounds;
 import org.opensearch.search.aggregations.metrics.ParsedGeoCentroid;
 import org.opensearch.search.aggregations.metrics.ParsedHDRPercentileRanks;
 import org.opensearch.search.aggregations.metrics.ParsedHDRPercentiles;
@@ -2240,7 +2234,6 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c));
         map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c));
         map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c));
-        map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c));
         map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
         map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
         map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
@@ -2257,8 +2250,6 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c));
         map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
         map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
-        map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
-        map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c));
         map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
         map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
         map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=97a52d145762adc241bad7fd18289bf7f6801e08ece6badf80402fe2b9f250b1
+distributionSha256Sum=db9c8211ed63f61f60292c69e80d89196f9eb36665e369e7f00ac4cc841c2219

modules/geo/build.gradle

@@ -28,18 +28,25 @@
  * under the License.
  */
 apply plugin: 'opensearch.yaml-rest-test'
+apply plugin: 'opensearch.internal-cluster-test'
 
 opensearchplugin {
-  description 'Placeholder plugin for geospatial features in OpenSearch. only registers geo_shape field mapper for now'
+  description 'Plugin for geospatial features in OpenSearch. Registering the geo_shape and aggregations GeoBounds on Geo_Shape and Geo_Point'
   classname 'org.opensearch.geo.GeoModulePlugin'
 }
 
 restResources {
   restApi {
-    includeCore '_common', 'indices', 'index', 'search'
+    includeCore '_common', 'indices', 'index', 'search', 'bulk'
   }
 }
 artifacts {
   restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test'))
 }
-test.enabled = false
+/**
+ * These compiler arguments needs to be removed, as there are raw types being used in the GeoGrid and GeoTile aggregations.
+ */
+tasks.withType(JavaCompile).configureEach {
+  options.compilerArgs -= '-Xlint:rawtypes'
+  options.compilerArgs -= '-Xlint:unchecked'
+}

modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java

@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.geo;
+
+import org.opensearch.index.mapper.GeoShapeFieldMapper;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.TestGeoShapeFieldMapperPlugin;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * This is the base class for all the Geo related integration tests. Use this class to add the features and settings
+ * for the test cluster on which integration tests are running.
+ */
+public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCase {
+    /**
+     * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this
+     * geo plugin is not getting packaged in a zip, we need to load it before the tests run.
+     *
+     * @return List of {@link Plugin}
+     */
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(GeoModulePlugin.class);
+    }
+
+    /**
+     * This was added as a backdoor to Mock the implementation of {@link GeoShapeFieldMapper} which was coming from
+     * {@link GeoModulePlugin}. Mock implementation is {@link TestGeoShapeFieldMapperPlugin}. Now we are using the
+     * {@link GeoModulePlugin} in our integration tests we need to override this functionality to avoid multiple mapper
+     * error.
+     *
+     * @return boolean
+     */
+    @Override
+    protected boolean addMockGeoShapeFieldMapper() {
+        return false;
+    }
+}

modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java

@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.geo.search;
+
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.geo.GeoModulePluginIntegTestCase;
+import org.opensearch.geo.search.aggregations.metrics.GeoBounds;
+import org.opensearch.geo.tests.common.AggregationBuilders;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.closeTo;
+
+@OpenSearchIntegTestCase.SuiteScopeTestCase
+public class MissingValueIT extends GeoModulePluginIntegTestCase {
+
+    @Override
+    protected void setupSuiteScopeCluster() throws Exception {
+        assertAcked(prepareCreate("idx").setMapping("date", "type=date", "location", "type=geo_point", "str", "type=keyword").get());
+        indexRandom(
+            true,
+            client().prepareIndex("idx").setId("1").setSource(),
+            client().prepareIndex("idx")
+                .setId("2")
+                .setSource("str", "foo", "long", 3L, "double", 5.5, "date", "2015-05-07", "location", "1,2")
+        );
+    }
+
+    public void testUnmappedGeoBounds() {
+        SearchResponse response = client().prepareSearch("idx")
+            .addAggregation(AggregationBuilders.geoBounds("bounds").field("non_existing_field").missing("2,1"))
+            .get();
+        assertSearchResponse(response);
+        GeoBounds bounds = response.getAggregations().get("bounds");
+        assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5));
+        assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5));
+        assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5));
+        assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5));
+    }
+
+    public void testGeoBounds() {
+        SearchResponse response = client().prepareSearch("idx")
+            .addAggregation(AggregationBuilders.geoBounds("bounds").field("location").missing("2,1"))
+            .get();
+        assertSearchResponse(response);
+        GeoBounds bounds = response.getAggregations().get("bounds");
+        assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5));
+        assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5));
+        assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5));
+        assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5));
+    }
+}