Query Grouping Integration Tests (#85)

* Add Query Grouping Integtests

Signed-off-by: Siddhant Deshmukh <[email protected]>

* Spotless apply

Signed-off-by: Siddhant Deshmukh <[email protected]>

* Fix settings name in tests

Signed-off-by: Siddhant Deshmukh <[email protected]>

---------

Signed-off-by: Siddhant Deshmukh <[email protected]>
(cherry picked from commit 90a3450)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

src/test/java/org/opensearch/plugin/insights/QueryInsightsRestTestCase.java

@@ -9,10 +9,13 @@
 package org.opensearch.plugin.insights;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
@@ -167,8 +170,21 @@ protected String defaultTopQueriesSettings() {
         return "{\n"
             + "    \"persistent\" : {\n"
             + "        \"search.insights.top_queries.latency.enabled\" : \"true\",\n"
-            + "        \"search.insights.top_queries.latency.window_size\" : \"600s\",\n"
-            + "        \"search.insights.top_queries.latency.top_n_size\" : 5\n"
+            + "        \"search.insights.top_queries.latency.window_size\" : \"1m\",\n"
+            + "        \"search.insights.top_queries.latency.top_n_size\" : 5,\n"
+            + "        \"search.insights.top_queries.group_by\" : \"none\"\n"
+            + "    }\n"
+            + "}";
+    }
+
+    protected String defaultTopQueryGroupingSettings() {
+        return "{\n"
+            + "    \"persistent\" : {\n"
+            + "        \"search.insights.top_queries.latency.enabled\" : \"true\",\n"
+            + "        \"search.insights.top_queries.latency.window_size\" : \"1m\",\n"
+            + "        \"search.insights.top_queries.latency.top_n_size\" : 5,\n"
+            + "        \"search.insights.top_queries.group_by\" : \"similarity\",\n"
+            + "        \"search.insights.top_queries.max_groups_excluding_topn\" : 5\n"
             + "    }\n"
             + "}";
     }
@@ -196,4 +212,95 @@ protected void doSearch(int times) throws IOException {
             Assert.assertEquals(200, response.getStatusLine().getStatusCode());
         }
     }
+
+    protected void doSearch(String queryType, int times) throws IOException {
+        for (int i = 0; i < times; i++) {
+            // Do Search
+            Request request = new Request("GET", "/my-index-0/_search?size=20&pretty");
+
+            // Set query based on the query type
+            request.setJsonEntity(searchBody(queryType));
+
+            Response response = client().performRequest(request);
+            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        }
+    }
+
+    private String searchBody(String queryType) {
+        switch (queryType) {
+            case "match":
+                // Query shape 1: Match query
+                return "{\n" + "  \"query\": {\n" + "    \"match\": {\n" + "      \"field1\": \"value1\"\n" + "    }\n" + "  }\n" + "}";
+
+            case "range":
+                // Query shape 2: Range query
+                return "{\n"
+                    + "  \"query\": {\n"
+                    + "    \"range\": {\n"
+                    + "      \"field2\": {\n"
+                    + "        \"gte\": 10,\n"
+                    + "        \"lte\": 50\n"
+                    + "      }\n"
+                    + "    }\n"
+                    + "  }\n"
+                    + "}";
+
+            case "term":
+                // Query shape 3: Term query
+                return "{\n"
+                    + "  \"query\": {\n"
+                    + "    \"term\": {\n"
+                    + "      \"field3\": {\n"
+                    + "        \"value\": \"exact-value\"\n"
+                    + "      }\n"
+                    + "    }\n"
+                    + "  }\n"
+                    + "}";
+
+            default:
+                throw new IllegalArgumentException("Unknown query type: " + queryType);
+        }
+    }
+
+    protected int countTopQueries(String json) {
+        // Basic pattern to match JSON array elements in `top_queries`
+        Pattern pattern = Pattern.compile("\\{\\s*\"timestamp\"");
+        Matcher matcher = pattern.matcher(json);
+
+        int count = 0;
+        while (matcher.find()) {
+            count++;
+        }
+
+        return count;
+    }
+
+    protected void waitForEmptyTopQueriesResponse() throws IOException, InterruptedException {
+        boolean isEmpty = false;
+        long timeoutMillis = 70000; // 70 seconds timeout
+        long startTime = System.currentTimeMillis();
+
+        while (!isEmpty && (System.currentTimeMillis() - startTime) < timeoutMillis) {
+            Request request = new Request("GET", "/_insights/top_queries?pretty");
+            Response response = client().performRequest(request);
+
+            if (response.getStatusLine().getStatusCode() != 200) {
+                Thread.sleep(1000); // Sleep before retrying
+                continue;
+            }
+
+            String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
+
+            if (countTopQueries(responseBody) == 0) {
+                isEmpty = true;
+            } else {
+                Thread.sleep(1000); // Sleep before retrying
+            }
+        }
+
+        if (!isEmpty) {
+            throw new IllegalStateException("Top queries response did not become empty within the timeout period");
+        }
+    }
+
 }

src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperByNoneIT.java

@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.plugin.insights.core.service.grouper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.junit.Assert;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
+import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
+
+/**
+ * ITs for Grouping Top Queries by none
+ */
+public class MinMaxQueryGrouperByNoneIT extends QueryInsightsRestTestCase {
+
+    /**
+     * Grouping by none should not group queries
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    public void testGroupingByNone() throws IOException, InterruptedException {
+
+        waitForEmptyTopQueriesResponse();
+
+        // Enable top N feature and grouping by none
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity(groupByNoneSettings());
+        Response response = client().performRequest(request);
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        // Ensure records are drained to the top queries service
+        Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
+
+        // run five times to make sure the records are drained to the top queries services
+        for (int i = 0; i < 5; i++) {
+            // Get Top Queries and validate
+            request = new Request("GET", "/_insights/top_queries?pretty");
+            response = client().performRequest(request);
+            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+            String top_requests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
+
+            int top_n_array_size = countTopQueries(top_requests);
+
+            // Validate that all queries are listed separately (no grouping)
+            Assert.assertEquals(12, top_n_array_size);
+        }
+    }
+
+    private String groupByNoneSettings() {
+        return "{\n"
+            + "    \"persistent\" : {\n"
+            + "        \"search.insights.top_queries.latency.enabled\" : \"true\",\n"
+            + "        \"search.insights.top_queries.latency.window_size\" : \"1m\",\n"
+            + "        \"search.insights.top_queries.latency.top_n_size\" : 100,\n"
+            + "        \"search.insights.top_queries.group_by\" : \"none\",\n"
+            + "        \"search.insights.top_queries.max_groups_excluding_topn\" : 5\n"
+            + "    }\n"
+            + "}";
+    }
+}

src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperBySimilarityIT.java

@@ -0,0 +1,152 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.plugin.insights.core.service.grouper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.junit.Assert;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.client.ResponseException;
+import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
+import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
+
+/**
+ * ITs for Grouping Top Queries by similarity
+ */
+public class MinMaxQueryGrouperBySimilarityIT extends QueryInsightsRestTestCase {
+
+    /**
+     * test grouping top queries
+     *
+     * @throws IOException IOException
+     */
+    public void testGroupingBySimilarity() throws IOException, InterruptedException {
+
+        waitForEmptyTopQueriesResponse();
+
+        // Enable top N feature and grouping feature
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity(defaultTopQueryGroupingSettings());
+        Response response = client().performRequest(request);
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+
+        // Search
+        doSearch("range", 2);
+        doSearch("match", 6);
+        doSearch("term", 4);
+
+        // run five times to make sure the records are drained to the top queries services
+        for (int i = 0; i < 5; i++) {
+            // Get Top Queries
+            request = new Request("GET", "/_insights/top_queries?pretty");
+            response = client().performRequest(request);
+            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+
+            String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
+
+            // Extract and count top_queries
+            int topNArraySize = countTopQueries(responseBody);
+
+            if (topNArraySize == 0) {
+                Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
+                continue;
+            }
+
+            Assert.assertEquals(3, topNArraySize);
+        }
+    }
+
+    /**
+     * Test invalid query grouping settings
+     *
+     * @throws IOException IOException
+     */
+    public void testInvalidQueryGroupingSettings() throws IOException {
+        for (String setting : invalidQueryGroupingSettings()) {
+            Request request = new Request("PUT", "/_cluster/settings");
+            request.setJsonEntity(setting);
+            try {
+                client().performRequest(request);
+                fail("Should not succeed with invalid query grouping settings");
+            } catch (ResponseException e) {
+                assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
+            }
+        }
+    }
+
+    /**
+     * Test valid query grouping settings
+     *
+     * @throws IOException IOException
+     */
+    public void testValidQueryGroupingSettings() throws IOException {
+        for (String setting : validQueryGroupingSettings()) {
+            Request request = new Request("PUT", "/_cluster/settings");
+            request.setJsonEntity(setting);
+            Response response = client().performRequest(request);
+            assertEquals(200, response.getStatusLine().getStatusCode());
+        }
+    }
+
+    private String[] invalidQueryGroupingSettings() {
+        return new String[] {
+            // Invalid max_groups: below minimum (-1)
+            "{\n"
+                + "    \"persistent\" : {\n"
+                + "        \"search.insights.top_queries.max_groups_excluding_topn\" : -1\n"
+                + "    }\n"
+                + "}",
+
+            // Invalid max_groups: above maximum (10001)
+            "{\n"
+                + "    \"persistent\" : {\n"
+                + "        \"search.insights.top_queries.max_groups_excluding_topn\" : 10001\n"
+                + "    }\n"
+                + "}",
+
+            // Invalid group_by: unsupported value
+            "{\n"
+                + "    \"persistent\" : {\n"
+                + "        \"search.insights.top_queries.group_by\" : \"unsupported_value\"\n"
+                + "    }\n"
+                + "}" };
+    }
+
+    private String[] validQueryGroupingSettings() {
+        return new String[] {
+            // Valid max_groups: minimum value (0)
+            "{\n"
+                + "    \"persistent\" : {\n"
+                + "        \"search.insights.top_queries.max_groups_excluding_topn\" : 0\n"
+                + "    }\n"
+                + "}",
+
+            // Valid max_groups: maximum value (10000)
+            "{\n"
+                + "    \"persistent\" : {\n"
+                + "        \"search.insights.top_queries.max_groups_excluding_topn\" : 10000\n"
+                + "    }\n"
+                + "}",
+
+            // Valid group_by: supported value (SIMILARITY)
+            "{\n" + "    \"persistent\" : {\n" + "        \"search.insights.top_queries.group_by\" : \"SIMILARITY\"\n" + "    }\n" + "}" };
+    }
+
+    private String groupByNoneSettings() {
+        return "{\n"
+            + "    \"persistent\" : {\n"
+            + "        \"search.insights.top_queries.latency.enabled\" : \"true\",\n"
+            + "        \"search.insights.top_queries.latency.window_size\" : \"1m\",\n"
+            + "        \"search.insights.top_queries.latency.top_n_size\" : 100,\n"
+            + "        \"search.insights.top_queries.group_by\" : \"none\",\n"
+            + "        \"search.insights.top_queries.max_groups_excluding_topn\" : 5\n"
+            + "    }\n"
+            + "}";
+    }
+}