Add additional grouping ITs and refactor

src/test/java/org/opensearch/plugin/insights/QueryInsightsRestTestCase.java

@@ -14,6 +14,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Supplier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -45,6 +46,7 @@
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
 public abstract class QueryInsightsRestTestCase extends OpenSearchRestTestCase {
@@ -320,4 +322,60 @@ protected void waitForEmptyTopQueriesResponse() throws IOException, InterruptedE
         }
     }
 
+    protected void assertTopQueriesCount(int expectedTopQueriesCount, String type) throws IOException, InterruptedException {
+        // Ensure records are drained to the top queries service
+        Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
+
+        // run five times to make sure the records are drained to the top queries services
+        for (int i = 0; i < 5; i++) {
+            String responseBody = getTopQueries(type);
+
+            int topNArraySize = countTopQueries(responseBody);
+
+            if (topNArraySize < expectedTopQueriesCount) {
+                // Ensure records are drained to the top queries service
+                Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
+                continue;
+            }
+
+            // Validate that all queries are listed separately (no grouping)
+            Assert.assertEquals(expectedTopQueriesCount, topNArraySize);
+        }
+    }
+
+    protected String getTopQueries(String type) throws IOException {
+        // Base URL
+        String endpoint = "/_insights/top_queries?pretty";
+
+        if (type != null) {
+            switch (type) {
+                case "cpu":
+                case "memory":
+                case "latency":
+                    endpoint = "/_insights/top_queries?type=" + type + "&pretty";
+                    break;
+                case "all":
+                    // Keep the default endpoint (no type parameter)
+                    break;
+                default:
+                    // Throw an exception if the type is invalid
+                    throw new IllegalArgumentException("Invalid type: " + type + ". Valid types are 'all', 'cpu', 'memory', or 'latency'.");
+            }
+        }
+
+        Request request = new Request("GET", endpoint);
+        Response response = client().performRequest(request);
+
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+
+        String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
+        return responseBody;
+    }
+
+    protected void updateClusterSettings(Supplier<String> settingsSupplier) throws IOException {
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity(settingsSupplier.get());
+        Response response = client().performRequest(request);
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+    }
 }

src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperByNoneIT.java

@@ -8,12 +8,7 @@
 package org.opensearch.plugin.insights.core.service.grouper;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import org.junit.Assert;
-import org.opensearch.client.Request;
-import org.opensearch.client.Response;
 import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
 
 /**
  * ITs for Grouping Top Queries by none
@@ -29,33 +24,14 @@ public void testGroupingByNone() throws IOException, InterruptedException {
 
         waitForEmptyTopQueriesResponse();
 
-        // Enable top N feature and grouping by none
-        Request request = new Request("PUT", "/_cluster/settings");
-        request.setJsonEntity(groupByNoneSettings());
-        Response response = client().performRequest(request);
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        updateClusterSettings(this::groupByNoneSettings);
 
         // Search
         doSearch("range", 2);
         doSearch("match", 6);
         doSearch("term", 4);
 
-        // Ensure records are drained to the top queries service
-        Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
-
-        // run five times to make sure the records are drained to the top queries services
-        for (int i = 0; i < 5; i++) {
-            // Get Top Queries and validate
-            request = new Request("GET", "/_insights/top_queries?pretty");
-            response = client().performRequest(request);
-            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-            String top_requests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
-
-            int top_n_array_size = countTopQueries(top_requests);
-
-            // Validate that all queries are listed separately (no grouping)
-            Assert.assertEquals(12, top_n_array_size);
-        }
+        assertTopQueriesCount(12, "latency");
     }
 
     private String groupByNoneSettings() {

src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperBySimilarityIT.java

@@ -8,13 +8,10 @@
 package org.opensearch.plugin.insights.core.service.grouper;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import org.junit.Assert;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
 import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
 
 /**
  * ITs for Grouping Top Queries by similarity
@@ -30,36 +27,14 @@ public void testGroupingBySimilarity() throws IOException, InterruptedException
 
         waitForEmptyTopQueriesResponse();
 
-        // Enable top N feature and grouping feature
-        Request request = new Request("PUT", "/_cluster/settings");
-        request.setJsonEntity(defaultTopQueryGroupingSettings());
-        Response response = client().performRequest(request);
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        updateClusterSettings(this::defaultTopQueryGroupingSettings);
 
         // Search
         doSearch("range", 2);
         doSearch("match", 6);
         doSearch("term", 4);
 
-        // run five times to make sure the records are drained to the top queries services
-        for (int i = 0; i < 5; i++) {
-            // Get Top Queries
-            request = new Request("GET", "/_insights/top_queries?pretty");
-            response = client().performRequest(request);
-            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-
-            String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
-
-            // Extract and count top_queries
-            int topNArraySize = countTopQueries(responseBody);
-
-            if (topNArraySize == 0) {
-                Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
-                continue;
-            }
-
-            Assert.assertEquals(3, topNArraySize);
-        }
+        assertTopQueriesCount(3, "latency");
     }
 
     /**

src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperIT.java

@@ -0,0 +1,106 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.plugin.insights.core.service.grouper;
+
+import java.io.IOException;
+import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
+
+public class MinMaxQueryGrouperIT extends QueryInsightsRestTestCase {
+    /**
+     * Grouping by none should not group queries
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    public void testNoneToSimilarityGroupingTransition() throws IOException, InterruptedException {
+
+        waitForEmptyTopQueriesResponse();
+
+        updateClusterSettings(this::defaultTopQueriesSettings);
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        assertTopQueriesCount(12, "latency");
+
+        updateClusterSettings(this::defaultTopQueryGroupingSettings);
+
+        // Top queries should be drained due to grouping change from NONE -> SIMILARITY
+        assertTopQueriesCount(0, "latency");
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        // 3 groups
+        assertTopQueriesCount(3, "latency");
+    }
+
+    public void testSimilarityToNoneGroupingTransition() throws IOException, InterruptedException {
+
+        waitForEmptyTopQueriesResponse();
+
+        updateClusterSettings(this::defaultTopQueryGroupingSettings);
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        assertTopQueriesCount(3, "latency");
+
+        updateClusterSettings(this::defaultTopQueriesSettings);
+
+        // Top queries should be drained due to grouping change from SIMILARITY -> NONE
+        assertTopQueriesCount(0, "latency");
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        assertTopQueriesCount(12, "latency");
+    }
+
+    public void testSimilarityMaxGroupsChanged() throws IOException, InterruptedException {
+
+        waitForEmptyTopQueriesResponse();
+
+        updateClusterSettings(this::defaultTopQueryGroupingSettings);
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        assertTopQueriesCount(3, "latency");
+
+        // Change max groups exluding topn setting
+        updateClusterSettings(this::updateMaxGroupsExcludingTopNSetting);
+
+        // Top queries should be drained due to max group change
+        assertTopQueriesCount(0, "latency");
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        assertTopQueriesCount(3, "latency");
+    }
+
+    protected String updateMaxGroupsExcludingTopNSetting() {
+        return "{\n"
+            + "    \"persistent\" : {\n"
+            + "        \"search.insights.top_queries.max_groups_excluding_topn\" : 1\n"
+            + "    }\n"
+            + "}";
+    }
+}

src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.java

@@ -9,7 +9,6 @@
 package org.opensearch.plugin.insights.rules.resthandler.top_queries;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
 import org.junit.Assert;
@@ -20,7 +19,6 @@
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
-import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
 
 /** Rest Action tests for Top Queries */
 public class TopQueriesRestIT extends QueryInsightsRestTestCase {
@@ -52,55 +50,19 @@ public void testTopQueriesResponses() throws IOException, InterruptedException {
         waitForEmptyTopQueriesResponse();
 
         // Enable Top N Queries feature
-        Request request = new Request("PUT", "/_cluster/settings");
-        request.setJsonEntity(defaultTopQueriesSettings());
-        Response response = client().performRequest(request);
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-        doSearch(2);
-        // run five times to make sure the records are drained to the top queries services
-        for (int i = 0; i < 5; i++) {
-            // Get Top Queries
-            request = new Request("GET", "/_insights/top_queries?pretty");
-            response = client().performRequest(request);
-            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-            String topRequests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
-            Assert.assertTrue(topRequests.contains("top_queries"));
+        updateClusterSettings(this::defaultTopQueriesSettings);
 
-            int topNArraySize = countTopQueries(topRequests);
+        doSearch(2);
 
-            if (topNArraySize == 0) {
-                Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
-                continue;
-            }
-            Assert.assertEquals(2, topNArraySize);
-        }
+        assertTopQueriesCount(2, "latency");
 
         // Enable Top N Queries by resource usage
-        request = new Request("PUT", "/_cluster/settings");
-        request.setJsonEntity(topQueriesByResourceUsagesSettings());
-        response = client().performRequest(request);
-        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        updateClusterSettings(this::topQueriesByResourceUsagesSettings);
+
         // Do Search
         doSearch(2);
 
-        // Run five times to make sure the records are drained to the top queries services
-        for (int i = 0; i < 5; i++) {
-            // Get Top Queries
-            request = new Request("GET", "/_insights/top_queries?type=cpu&pretty");
-            response = client().performRequest(request);
-            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
-            String topRequests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
-            Assert.assertTrue(topRequests.contains("top_queries"));
-
-            // Use the countTopQueries method to determine the number of top queries
-            int topNArraySize = countTopQueries(topRequests);
-
-            if (topNArraySize == 0) {
-                Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
-                continue;
-            }
-            Assert.assertEquals(2, topNArraySize);
-        }
+        assertTopQueriesCount(2, "cpu");
     }
 
     /**
@@ -125,10 +87,10 @@ private String topQueriesByResourceUsagesSettings() {
         return "{\n"
             + "    \"persistent\" : {\n"
             + "        \"search.insights.top_queries.memory.enabled\" : \"true\",\n"
-            + "        \"search.insights.top_queries.memory.window_size\" : \"600s\",\n"
+            + "        \"search.insights.top_queries.memory.window_size\" : \"1m\",\n"
             + "        \"search.insights.top_queries.memory.top_n_size\" : \"5\",\n"
             + "        \"search.insights.top_queries.cpu.enabled\" : \"true\",\n"
-            + "        \"search.insights.top_queries.cpu.window_size\" : \"600s\",\n"
+            + "        \"search.insights.top_queries.cpu.window_size\" : \"1m\",\n"
             + "        \"search.insights.top_queries.cpu.top_n_size\" : 5,\n"
             + "        \"search.insights.top_queries.group_by\" : \"none\"\n"
             + "    }\n"