From be496bc763ac94660098c3a9e1f4a75167731789 Mon Sep 17 00:00:00 2001 From: Gonzalo Ortiz Jaureguizar Date: Thu, 7 Sep 2023 02:44:12 +0200 Subject: [PATCH] Multi stage int tests (#11404) * Run OfflineClusterIntegrationTest test with both query engines. * Improved some tests in V2 * Improve some tests in OfflineClusterIntegrationTest * Run more integration test with multi stage engine * Run OfflineClusterIntegrationTest test with both query engines. * Improved some tests in V2 * Fix MultiStageEngineIntegrationTest * Apply some changes in BaseClusterIntegrationTestSet * Extra improvements in OfflineClusterIntegrationTest * Adapted error code expected on a specific query * Fix or skip all tests in OfflineClusterIntegrationTest * Skip all tests that fail in BaseRealtimeClusterIntegrationTest * Skip tests that doesn't work in V2 using a specific function * Fix MultiStageEngineIntegrationTest to run with the correct transport * Add a TODO * Fix several error codes * Enable one now supported V2 tests --------- Co-authored-by: Xiang Fu --- .../pinot/client/ConnectionFactory.java | 1 + .../tests/BaseClusterIntegrationTest.java | 31 +- .../pinot/integration/tests/ClusterTest.java | 7 + ...ggregateMetricsClusterIntegrationTest.java | 5 +- .../tests/BaseClusterIntegrationTestSet.java | 6 + .../BaseRealtimeClusterIntegrationTest.java | 34 +- .../GrpcBrokerClusterIntegrationTest.java | 5 +- .../tests/HybridClusterIntegrationTest.java | 56 +- .../IngestionConfigHybridIntegrationTest.java | 5 +- .../LLCRealtimeClusterIntegrationTest.java | 6 +- ...ltiNodesOfflineClusterIntegrationTest.java | 9 +- .../MultiStageEngineIntegrationTest.java | 49 +- .../tests/NullHandlingIntegrationTest.java | 76 ++- .../tests/OfflineClusterIntegrationTest.java | 576 +++++++++++++----- ...ClusterMemBasedServerQueryKillingTest.java | 18 +- ...eClusterServerCPUTimeQueryKillingTest.java | 6 +- ...neGRPCServerMultiStageIntegrationTest.java | 29 + .../tests/StarTreeClusterIntegrationTest.java | 10 +- .../tests/custom/TextIndicesTest.java | 10 +- 19 files changed, 691 insertions(+), 248 deletions(-) create mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java diff --git a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ConnectionFactory.java b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ConnectionFactory.java index 1ed65dd2ebcf..45a92be48b60 100644 --- a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ConnectionFactory.java +++ b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ConnectionFactory.java @@ -226,6 +226,7 @@ public static Connection fromHostList(Properties properties, List broker } private static PinotClientTransport getDefault(Properties connectionProperties) { + // TODO: This code incorrectly assumes that connection properties are always the same if (_defaultTransport == null) { synchronized (ConnectionFactory.class) { if (_defaultTransport == null) { diff --git a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java index fc50b2d93187..6b6cc9202cc6 100644 --- a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java +++ b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.apache.pinot.integration.tests; +import com.fasterxml.jackson.databind.JsonNode; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -32,9 +33,11 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.function.Function; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.apache.pinot.client.ConnectionFactory; +import org.apache.pinot.client.JsonAsyncHttpPinotClientTransportFactory; import org.apache.pinot.client.ResultSetGroup; import org.apache.pinot.common.utils.TarGzCompressionUtils; import org.apache.pinot.common.utils.config.TagNameUtils; @@ -509,17 +512,24 @@ protected TableConfig getRealtimeTableConfig() { * @return Pinot connection */ protected org.apache.pinot.client.Connection getPinotConnection() { + // TODO: This code is assuming getPinotConnectionProperties() will always return the same values if (useMultiStageQueryEngine()) { if (_pinotConnectionV2 == null) { Properties properties = getPinotConnectionProperties(); properties.put("useMultiStageEngine", "true"); - _pinotConnectionV2 = ConnectionFactory.fromZookeeper(properties, getZkUrl() + "/" + getHelixClusterName()); + _pinotConnectionV2 = ConnectionFactory.fromZookeeper(getZkUrl() + "/" + getHelixClusterName(), + new JsonAsyncHttpPinotClientTransportFactory() + .withConnectionProperties(properties) + .buildTransport()); } return _pinotConnectionV2; } if (_pinotConnection == null) { _pinotConnection = - ConnectionFactory.fromZookeeper(getPinotConnectionProperties(), getZkUrl() + "/" + getHelixClusterName()); + ConnectionFactory.fromZookeeper(getZkUrl() + "/" + getHelixClusterName(), + new JsonAsyncHttpPinotClientTransportFactory() + .withConnectionProperties(getPinotConnectionProperties()) + .buildTransport()); } return _pinotConnection; } @@ -789,4 +799,21 @@ protected void testQueryWithMatchingRowCount(String pinotQuery, String h2Query) ClusterIntegrationTestUtils.testQueryWithMatchingRowCount(pinotQuery, getBrokerBaseApiUrl(), getPinotConnection(), h2Query, getH2Connection(), null, getExtraQueryProperties(), useMultiStageQueryEngine()); } + + protected String getType(JsonNode jsonNode, int colIndex) { + return jsonNode.get("resultTable") + .get("dataSchema") + .get("columnDataTypes") + .get(colIndex) + .asText(); + } + + protected T getCellValue(JsonNode jsonNode, int colIndex, int rowIndex, Function extract) { + JsonNode cellResult = jsonNode.get("resultTable").get("rows").get(rowIndex).get(colIndex); + return extract.apply(cellResult); + } + + protected long getLongCellValue(JsonNode jsonNode, int colIndex, int rowIndex) { + return getCellValue(jsonNode, colIndex, rowIndex, JsonNode::asLong).longValue(); + } } diff --git a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java index 9845318f30ca..9b41232f9b96 100644 --- a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java +++ b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java @@ -78,6 +78,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; +import org.testng.SkipException; import org.testng.annotations.DataProvider; import static org.apache.pinot.integration.tests.ClusterIntegrationTestUtils.getBrokerQueryApiUrl; @@ -572,4 +573,10 @@ public Object[][] useV2QueryEngine() { {true} }; } + + protected void notSupportedInV2() { + if (useMultiStageQueryEngine()) { + throw new SkipException("Some queries fail when using multi-stage engine"); + } + } } diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/AggregateMetricsClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/AggregateMetricsClusterIntegrationTest.java index 5fd41d9d95af..7ad116a8c73a 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/AggregateMetricsClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/AggregateMetricsClusterIntegrationTest.java @@ -99,9 +99,10 @@ protected void waitForAllDocsLoaded(long timeoutMs) { }, 100L, timeoutMs, "Failed to load all documents"); } - @Test - public void testQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query = "SELECT SUM(AirTime), SUM(ArrDelay) FROM mytable"; testQuery(query); query = "SELECT SUM(AirTime), DaysSinceEpoch FROM mytable GROUP BY DaysSinceEpoch ORDER BY SUM(AirTime) DESC"; diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java index 5b07d1cc37e2..f757ac54deab 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java @@ -48,6 +48,7 @@ import org.apache.pinot.util.TestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.annotations.BeforeMethod; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -67,6 +68,11 @@ public abstract class BaseClusterIntegrationTestSet extends BaseClusterIntegrati "On_Time_On_Time_Performance_2014_100k_subset.test_queries_200.sql"; private static final int DEFAULT_NUM_QUERIES_TO_GENERATE = 100; + @BeforeMethod + public void resetMultiStage() { + setUseMultiStageQueryEngine(false); + } + /** * Can be overridden to change default setting */ diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java index 83adb4dfe9f0..af39f3280e37 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java @@ -106,9 +106,10 @@ protected List getNoDictionaryColumns() { * to ensure the right result is computed, wherein dictionary is not read if it is mutable * @throws Exception */ - @Test - public void testDictionaryBasedQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testDictionaryBasedQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // Dictionary columns // int @@ -142,30 +143,35 @@ private void testDictionaryBasedFunctions(String column) String.format("SELECT MAX(%s)-MIN(%s) FROM %s", column, column, getTableName())); } - @Test - public void testHardcodedQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testHardcodedQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); super.testHardcodedQueries(); } - @Test - @Override - public void testQueriesFromQueryFile() + @Test(dataProvider = "useBothQueryEngines") + public void testQueriesFromQueryFile(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); super.testQueriesFromQueryFile(); } - @Test - @Override - public void testGeneratedQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testGeneratedQueries(boolean useMultiStageQueryEngine) throws Exception { - testGeneratedQueries(true, false); + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); + testGeneratedQueries(true, useMultiStageQueryEngine); } - @Test - @Override - public void testQueryExceptions() + @Test(dataProvider = "useBothQueryEngines") + public void testQueryExceptions(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); super.testQueryExceptions(); } diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/GrpcBrokerClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/GrpcBrokerClusterIntegrationTest.java index a0dc29c61cb2..55cccc064b4e 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/GrpcBrokerClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/GrpcBrokerClusterIntegrationTest.java @@ -119,9 +119,10 @@ protected void startHybridCluster() createServerTenant(TENANT_NAME, 1, 1); } - @Test - public void testGrpcBrokerRequestHandlerOnSelectionOnlyQuery() + @Test(dataProvider = "useBothQueryEngines") + public void testGrpcBrokerRequestHandlerOnSelectionOnlyQuery(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query = "SELECT * FROM mytable LIMIT 1000000"; testQuery(query); query = "SELECT * FROM mytable WHERE DaysSinceEpoch > 16312 LIMIT 10000000"; diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java index f366a994eba9..2b5c36bc59f9 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java @@ -185,9 +185,10 @@ public void testBrokerDebugOutput() Assert.assertNotNull(getDebugInfo("debug/routingTable/" + TableNameBuilder.REALTIME.tableNameWithType(tableName))); } - @Test - public void testBrokerDebugRoutingTableSQL() + @Test(dataProvider = "useBothQueryEngines") + public void testBrokerDebugRoutingTableSQL(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String tableName = getTableName(); String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName); String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName); @@ -198,9 +199,11 @@ public void testBrokerDebugRoutingTableSQL() Assert.assertNotNull(getDebugInfo("debug/routingTable/sql?query=" + encodedSQL)); } - @Test - public void testQueryTracing() + @Test(dataProvider = "useBothQueryEngines") + public void testQueryTracing(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); JsonNode jsonNode = postQuery("SET trace = true; SELECT COUNT(*) FROM " + getTableName()); Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asLong(), getCountStarResult()); Assert.assertTrue(jsonNode.get("exceptions").isEmpty()); @@ -210,9 +213,11 @@ public void testQueryTracing() Assert.assertTrue(traceInfo.has("localhost_R")); } - @Test - public void testQueryTracingWithLiteral() + @Test(dataProvider = "useBothQueryEngines") + public void testQueryTracingWithLiteral(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); JsonNode jsonNode = postQuery("SET trace = true; SELECT 1, \'test\', ArrDelay FROM " + getTableName() + " LIMIT 10"); long countStarResult = 10; @@ -228,9 +233,10 @@ public void testQueryTracingWithLiteral() Assert.assertTrue(traceInfo.has("localhost_R")); } - @Test - public void testDropResults() + @Test(dataProvider = "useBothQueryEngines") + public void testDropResults(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); final String query = String.format("SELECT * FROM %s limit 10", getTableName()); final String resultTag = "resultTable"; @@ -244,31 +250,35 @@ public void testDropResults() Assert.assertTrue(postQueryWithOptions(query, "dropResults=truee").has(resultTag)); } - @Test - @Override - public void testHardcodedQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testHardcodedQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); super.testHardcodedQueries(); } - @Test - @Override - public void testQueriesFromQueryFile() + @Test(dataProvider = "useBothQueryEngines") + public void testQueriesFromQueryFile(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); super.testQueriesFromQueryFile(); } - @Test - @Override - public void testGeneratedQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testGeneratedQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); super.testGeneratedQueries(); } - @Test - @Override - public void testQueryExceptions() + @Test(dataProvider = "useBothQueryEngines") + public void testQueryExceptions(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); super.testQueryExceptions(); } @@ -286,9 +296,9 @@ public void testBrokerResponseMetadata() super.testBrokerResponseMetadata(); } - @Test - @Override - public void testVirtualColumnQueries() { + @Test(dataProvider = "useBothQueryEngines") + public void testVirtualColumnQueries(boolean useMultiStageQueryEngine) + throws Exception { super.testVirtualColumnQueries(); } diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/IngestionConfigHybridIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/IngestionConfigHybridIntegrationTest.java index 918b15cae77f..7495be1e0f79 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/IngestionConfigHybridIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/IngestionConfigHybridIntegrationTest.java @@ -151,9 +151,10 @@ public void setUp() waitForAllDocsLoaded(600_000L); } - @Test - public void testQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // Select column created with transform function String sqlQuery = "Select millisSinceEpoch from " + DEFAULT_TABLE_NAME; JsonNode response = postQuery(sqlQuery); diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java index 7068656865a1..88fe46e4e16e 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java @@ -262,9 +262,11 @@ public void testReload() testReload(false); } - @Test - public void testAddRemoveDictionaryAndInvertedIndex() + @Test(dataProvider = "useBothQueryEngines") + public void testAddRemoveDictionaryAndInvertedIndex(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); String query = "SELECT COUNT(*) FROM myTable WHERE ActualElapsedTime = -9999"; long numTotalDocs = getCountStarResult(); diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java index 7c76eaaacb3a..f2b345a690f8 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiNodesOfflineClusterIntegrationTest.java @@ -194,21 +194,19 @@ private void testCountStarQuery(int expectedNumServersQueried, boolean exception // Disabled because with multiple replicas, there is no guarantee that all replicas are reloaded @Test(enabled = false) - @Override - public void testStarTreeTriggering() { + public void testStarTreeTriggering(boolean useMultiStageQueryEngine) { // Ignored } // Disabled because with multiple replicas, there is no guarantee that all replicas are reloaded @Test(enabled = false) @Override - public void testDefaultColumns() { + public void testDefaultColumns(boolean useMultiStageQueryEngineg) { // Ignored } // Disabled because with multiple replicas, there is no guarantee that all replicas are reloaded @Test(enabled = false) - @Override public void testBloomFilterTriggering() { // Ignored } @@ -216,7 +214,8 @@ public void testBloomFilterTriggering() { // Disabled because with multiple replicas, there is no guarantee that all replicas are reloaded @Test(enabled = false) @Override - public void testRangeIndexTriggering() { + public void testRangeIndexTriggering(boolean useMultiStageQueryEngine) + throws Exception { // Ignored } diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java index 4099b744c216..c18dd026ffac 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java @@ -36,6 +36,7 @@ import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import static org.apache.pinot.common.function.scalar.StringFunctions.*; @@ -91,9 +92,15 @@ protected void setupTenants() throws IOException { } +// @Override +// protected boolean useMultiStageQueryEngine() { +// return true; +// } + + @BeforeMethod @Override - protected boolean useMultiStageQueryEngine() { - return true; + public void resetMultiStage() { + setUseMultiStageQueryEngine(true); } @Test @@ -157,12 +164,21 @@ public void testDistinctCountQueries(boolean useMultiStageQueryEngine) double[] expectedNumericResults = new double[]{ 364, 364, 355, 364, 364, 364, 5915969, 16252.662087912087 }; + double[] expectedNumericResultsV1 = new double[]{ + 364, 364, 357, 364, 364, 364, 5915969, 16252.662087912087 + }; Assert.assertEquals(numericResultFunctions.length, expectedNumericResults.length); for (int i = 0; i < numericResultFunctions.length; i++) { String pinotQuery = String.format("SELECT %s(DaysSinceEpoch) FROM mytable", numericResultFunctions[i]); JsonNode jsonNode = postQuery(pinotQuery); - Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble(), expectedNumericResults[i]); + if (useMultiStageQueryEngine) { + Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble(), + expectedNumericResults[i]); + } else { + Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble(), + expectedNumericResultsV1[i]); + } } String[] binaryResultFunctions = new String[]{ @@ -172,14 +188,21 @@ public void testDistinctCountQueries(boolean useMultiStageQueryEngine) 360, 3904 }; + int[] expectedBinarySizeResultsV1 = new int[]{ + 5480, + 3904 + }; for (int i = 0; i < binaryResultFunctions.length; i++) { String pinotQuery = String.format("SELECT %s(DaysSinceEpoch) FROM mytable", binaryResultFunctions[i]); JsonNode jsonNode = postQuery(pinotQuery); - Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asText().length(), - expectedBinarySizeResults[i]); + if (useMultiStageQueryEngine) { + Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asText().length(), + expectedBinarySizeResults[i]); + } else { + Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asText().length(), + expectedBinarySizeResultsV1[i]); + } } - - setUseMultiStageQueryEngine(true); } @Test(dataProvider = "useBothQueryEngines") @@ -195,13 +218,21 @@ public void testMultiValueColumnAggregationQuery(boolean useMultiStageQueryEngin -5.421344202E9, 577725, -9999.0, 16271.0, -9383.95292223809, 26270.0, 312, 312, 328, 3954484.0, 12674.628205128205 }; + double[] expectedResultsV1 = new double[]{ + -5.421344202E9, 577725, -9999.0, 16271.0, -9383.95292223809, 26270.0, 312, 312, 312, 3954484.0, + 12674.628205128205 + }; Assert.assertEquals(multiValueFunctions.length, expectedResults.length); for (int i = 0; i < multiValueFunctions.length; i++) { String pinotQuery = String.format("SELECT %s(DivAirportIDs) FROM mytable", multiValueFunctions[i]); JsonNode jsonNode = postQuery(pinotQuery); - Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble(), expectedResults[i]); + if (useMultiStageQueryEngine) { + Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble(), expectedResults[i]); + } else { + Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble(), expectedResultsV1[i]); + } } String pinotQuery = "SELECT percentileMV(DivAirportIDs, 99) FROM mytable"; @@ -227,8 +258,6 @@ public void testMultiValueColumnAggregationQuery(boolean useMultiStageQueryEngin jsonNode = postQuery(pinotQuery); Assert.assertTrue(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble() > 10000); Assert.assertTrue(jsonNode.get("resultTable").get("rows").get(0).get(0).asDouble() < 17000); - - setUseMultiStageQueryEngine(true); } @Test diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java index cb2163b63263..3fc9e3e063a1 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NullHandlingIntegrationTest.java @@ -139,62 +139,72 @@ protected long getCountStarResult() { return 100; } - @Test - public void testTotalCount() + @Test(dataProvider = "useBothQueryEngines") + public void testTotalCount(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query = "SELECT COUNT(*) FROM " + getTableName(); testQuery(query); } - @Test - public void testCountWithNullDescription() + @Test(dataProvider = "useBothQueryEngines") + public void testCountWithNullDescription(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); String query = "SELECT COUNT(*) FROM " + getTableName() + " WHERE description IS NOT NULL"; testQuery(query); } - @Test - public void testCountWithNullDescriptionAndSalary() + @Test(dataProvider = "useBothQueryEngines") + public void testCountWithNullDescriptionAndSalary(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); String query = "SELECT COUNT(*) FROM " + getTableName() + " WHERE description IS NOT NULL AND salary IS NOT NULL"; testQuery(query); } - @Test - public void testCaseWithNullSalary() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseWithNullSalary(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query = "SELECT CASE WHEN salary IS NULL THEN 1 ELSE 0 END FROM " + getTableName(); testQuery(query); } - @Test - public void testCaseWithNotNullDescription() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseWithNotNullDescription(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query = "SELECT CASE WHEN description IS NOT NULL THEN 1 ELSE 0 END FROM " + getTableName(); testQuery(query); } - @Test - public void testCaseWithIsDistinctFrom() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseWithIsDistinctFrom(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query = "SELECT salary IS DISTINCT FROM salary FROM " + getTableName(); testQuery(query); query = "SELECT salary FROM " + getTableName() + " where salary IS DISTINCT FROM salary"; testQuery(query); } - @Test - public void testCaseWithIsNotDistinctFrom() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseWithIsNotDistinctFrom(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query = "SELECT description IS NOT DISTINCT FROM description FROM " + getTableName(); testQuery(query); query = "SELECT description FROM " + getTableName() + " where description IS NOT DISTINCT FROM description"; testQuery(query); } - @Test - public void testTotalCountWithNullHandlingQueryOptionEnabled() - throws Exception { + @Test(dataProvider = "useBothQueryEngines") + public void testTotalCountWithNullHandlingQueryOptionEnabled(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String pinotQuery = "SELECT COUNT(*) FROM " + getTableName() + " option(enableNullHandling=true)"; String h2Query = "SELECT COUNT(*) FROM " + getTableName(); testQuery(pinotQuery, h2Query); @@ -205,9 +215,11 @@ public void testTotalCountWithNullHandlingQueryOptionEnabled() DataTableBuilderFactory.setDataTableVersion(DataTableBuilderFactory.DEFAULT_VERSION); } - @Test - public void testNullLiteralSelectionOnlyBroker() + @Test(dataProvider = "useBothQueryEngines") + public void testNullLiteralSelectionOnlyBroker(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); // Null literal only String sqlQuery = "SELECT null FROM mytable OPTION(enableNullHandling=true)"; JsonNode response = postQuery(sqlQuery); @@ -300,35 +312,40 @@ public void testNullLiteralSelectionOnlyBroker() assertEquals(rows.get(0).get(0).asText(), "null"); } - @Test - public void testOrderByNullsFirst() + @Test(dataProvider = "useBothQueryEngines") + public void testOrderByNullsFirst(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String h2Query = "SELECT salary FROM " + getTableName() + " ORDER BY salary NULLS FIRST"; String pinotQuery = h2Query + " option(enableNullHandling=true)"; testQuery(pinotQuery, h2Query); } - @Test - public void testOrderByNullsLast() + @Test(dataProvider = "useBothQueryEngines") + public void testOrderByNullsLast(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String h2Query = "SELECT salary FROM " + getTableName() + " ORDER BY salary DESC NULLS LAST"; String pinotQuery = h2Query + " option(enableNullHandling=true)"; testQuery(pinotQuery, h2Query); } - @Test - public void testDistinctOrderByNullsLast() + @Test(dataProvider = "useBothQueryEngines") + public void testDistinctOrderByNullsLast(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String h2Query = "SELECT distinct salary FROM " + getTableName() + " ORDER BY salary DESC NULLS LAST"; String pinotQuery = h2Query + " option(enableNullHandling=true)"; testQuery(pinotQuery, h2Query); } - @Test - public void testSelectNullLiteral() throws Exception { + @Test(dataProvider = "useBothQueryEngines") + public void testSelectNullLiteral(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // Need to also select an identifier column to skip the all literal query optimization which returns without // querying the segment. String sqlQuery = "SELECT NULL, salary FROM mytable OPTION(enableNullHandling=true)"; @@ -339,9 +356,10 @@ public void testSelectNullLiteral() throws Exception { assertEquals(rows.get(0).get(0).asText(), "null"); } - @Test - public void testCaseWhenAllLiteral() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseWhenAllLiteral(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String sqlQuery = "SELECT CASE WHEN true THEN 1 WHEN NOT true THEN 0 ELSE NULL END FROM mytable OPTION(enableNullHandling=true)"; diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java index 8c0d6602b71b..0be7e109c634 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java @@ -39,6 +39,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.apache.helix.model.IdealState; @@ -273,6 +274,15 @@ protected void testQuery(String pinotQuery, String h2Query) super.testQuery(pinotQuery, h2Query); } + private void testQueryError(String query, int errorCode) + throws Exception { + JsonNode response = postQuery(query); + JsonNode exceptions = response.get("exceptions"); + assertFalse(exceptions.isEmpty(), "At least one exception was expected"); + JsonNode firstException = exceptions.get(0); + assertEquals(firstException.get("errorCode").asInt(), errorCode); + } + @Test public void testInstancesStarted() { assertEquals(_serviceStatusCallbacks.size(), getNumBrokers() + getNumServers()); @@ -573,9 +583,10 @@ private void addInvertedIndex() assertEquals(postQuery(TEST_UPDATED_INVERTED_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L); } - @Test - public void testTimeFunc() + @Test(dataProvider = "useBothQueryEngines") + public void testTimeFunc(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String sqlQuery = "SELECT toDateTime(now(), 'yyyy-MM-dd z'), toDateTime(ago('PT1H'), 'yyyy-MM-dd z') FROM mytable"; JsonNode response = postQuery(sqlQuery); String todayStr = response.get("resultTable").get("rows").get(0).get(0).asText(); @@ -589,9 +600,10 @@ public void testTimeFunc() assertEquals(oneHourAgoTodayStr, expectedOneHourAgoTodayStr); } - @Test - public void testRegexpReplace() + @Test(dataProvider = "useBothQueryEngines") + public void testRegexpReplace(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // Correctness tests of regexpReplace. // Test replace all. @@ -678,7 +690,7 @@ public void testRegexpReplace() assertEquals(result, "healthy, wealthy, stealthy and wise"); // Test in select clause with column values - sqlQuery = "SELECT regexpReplace(DestCityName, ' ', '', 0, -1, 'i') from myTable where OriginState = 'CA'"; + sqlQuery = "SELECT regexpReplace(DestCityName, ' ', '', 0, -1, 'i') from mytable where OriginState = 'CA'"; response = postQuery(sqlQuery); JsonNode rows = response.get("resultTable").get("rows"); for (int i = 0; i < rows.size(); i++) { @@ -687,20 +699,20 @@ public void testRegexpReplace() } // Test in where clause - sqlQuery = "SELECT count(*) from myTable where regexpReplace(originState, '[VC]A', 'TEST') = 'TEST'"; + sqlQuery = "SELECT count(*) from mytable where regexpReplace(OriginState, '[VC]A', 'TEST') = 'TEST'"; response = postQuery(sqlQuery); int count1 = response.get("resultTable").get("rows").get(0).get(0).asInt(); - sqlQuery = "SELECT count(*) from myTable where originState='CA' or originState='VA'"; + sqlQuery = "SELECT count(*) from mytable where OriginState='CA' or OriginState='VA'"; response = postQuery(sqlQuery); int count2 = response.get("resultTable").get("rows").get(0).get(0).asInt(); assertEquals(count1, count2); // Test nested transform sqlQuery = - "SELECT count(*) from myTable where contains(regexpReplace(originState, '(C)(A)', '$1TEST$2'), 'CTESTA')"; + "SELECT count(*) from mytable where contains(regexpReplace(OriginState, '(C)(A)', '$1TEST$2'), 'CTESTA')"; response = postQuery(sqlQuery); count1 = response.get("resultTable").get("rows").get(0).get(0).asInt(); - sqlQuery = "SELECT count(*) from myTable where originState='CA'"; + sqlQuery = "SELECT count(*) from mytable where OriginState='CA'"; response = postQuery(sqlQuery); count2 = response.get("resultTable").get("rows").get(0).get(0).asInt(); assertEquals(count1, count2); @@ -709,9 +721,8 @@ public void testRegexpReplace() @Test public void testCastMV() throws Exception { - // simple cast - String sqlQuery = "SELECT DivLongestGTimes, CAST(DivLongestGTimes as DOUBLE) from myTable LIMIT 100"; + String sqlQuery = "SELECT DivLongestGTimes, CAST(DivLongestGTimes as DOUBLE) from mytable LIMIT 100"; JsonNode response = postQuery(sqlQuery); JsonNode resultTable = response.get("resultTable"); JsonNode dataSchema = resultTable.get("dataSchema"); @@ -735,7 +746,7 @@ public void testCastMV() // nested cast sqlQuery = "SELECT DivAirportIDs, CAST(CAST(CAST(DivAirportIDs AS FLOAT) as INT) as STRING)," - + " DivTotalGTimes, CAST(CAST(DivTotalGTimes AS STRING) AS LONG) from myTable ORDER BY CARRIER LIMIT 100"; + + " DivTotalGTimes, CAST(CAST(DivTotalGTimes AS STRING) AS LONG) from mytable ORDER BY CARRIER LIMIT 100"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); @@ -770,11 +781,12 @@ public void testCastMV() } } - @Test - public void testUrlFunc() + @Test(dataProvider = "useBothQueryEngines") + public void testUrlFunc(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String sqlQuery = "SELECT encodeUrl('key1=value 1&key2=value@!$2&key3=value%3'), " - + "decodeUrl('key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253') FROM myTable"; + + "decodeUrl('key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253') FROM mytable"; JsonNode response = postQuery(sqlQuery); String encodedString = response.get("resultTable").get("rows").get(0).get(0).asText(); String expectedUrlStr = encodeUrl("key1=value 1&key2=value@!$2&key3=value%3"); @@ -785,12 +797,13 @@ public void testUrlFunc() assertEquals(decodedString, expectedUrlStr); } - @Test - public void testBase64Func() + @Test(dataProvider = "useBothQueryEngines") + public void testBase64Func(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // string literal - String sqlQuery = "SELECT toBase64(toUtf8('hello!')), " + "fromUtf8(fromBase64('aGVsbG8h')) FROM myTable"; + String sqlQuery = "SELECT toBase64(toUtf8('hello!')), " + "fromUtf8(fromBase64('aGVsbG8h')) FROM mytable"; JsonNode response = postQuery(sqlQuery); JsonNode resultTable = response.get("resultTable"); JsonNode dataSchema = resultTable.get("dataSchema"); @@ -807,7 +820,7 @@ public void testBase64Func() // long string literal encode sqlQuery = "SELECT toBase64(toUtf8('this is a long string that will encode to more than 76 characters using base64')) " - + "FROM myTable"; + + "FROM mytable"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); rows = resultTable.get("rows"); @@ -818,7 +831,7 @@ public void testBase64Func() // long string literal decode sqlQuery = "SELECT fromUtf8(fromBase64" + "('dGhpcyBpcyBhIGxvbmcgc3RyaW5nIHRoYXQgd2lsbCBlbmNvZGUgdG8gbW9yZSB0aGFuIDc2IGNoYXJhY3RlcnMgdXNpbmcgYmFzZTY0" - + "')) FROM myTable"; + + "')) FROM mytable"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); rows = resultTable.get("rows"); @@ -827,7 +840,7 @@ public void testBase64Func() "dGhpcyBpcyBhIGxvbmcgc3RyaW5nIHRoYXQgd2lsbCBlbmNvZGUgdG8gbW9yZSB0aGFuIDc2IGNoYXJhY3RlcnMgdXNpbmcgYmFzZTY0"))); // non-string literal - sqlQuery = "SELECT toBase64(toUtf8(123)), fromUtf8(fromBase64(toBase64(toUtf8(123)))), 123 FROM myTable"; + sqlQuery = "SELECT toBase64(toUtf8(123)), fromUtf8(fromBase64(toBase64(toUtf8(123)))), 123 FROM mytable"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); rows = resultTable.get("rows"); @@ -839,7 +852,7 @@ public void testBase64Func() // identifier sqlQuery = "SELECT Carrier, toBase64(toUtf8(Carrier)), fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))), " - + "fromBase64(toBase64(toUtf8(Carrier))) FROM myTable LIMIT 100"; + + "fromBase64(toBase64(toUtf8(Carrier))) FROM mytable LIMIT 100"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); @@ -856,27 +869,43 @@ public void testBase64Func() } // invalid argument - sqlQuery = "SELECT toBase64() FROM myTable"; - response = postQuery(sqlQuery); - assertTrue(response.get("exceptions").get(0).get("message").toString().startsWith("\"QueryExecutionError")); + sqlQuery = "SELECT toBase64() FROM mytable"; + if (useMultiStageQueryEngine) { + testQueryError(sqlQuery, QueryException.QUERY_PLANNING_ERROR_CODE); + } else { + response = postQuery(sqlQuery); + assertTrue(response.get("exceptions").get(0).get("message").toString().startsWith("\"QueryExecutionError")); + } // invalid argument - sqlQuery = "SELECT fromBase64() FROM myTable"; - response = postQuery(sqlQuery); - assertTrue(response.get("exceptions").get(0).get("message").toString().startsWith("\"QueryExecutionError")); + sqlQuery = "SELECT fromBase64() FROM mytable"; + if (useMultiStageQueryEngine) { + testQueryError(sqlQuery, QueryException.QUERY_PLANNING_ERROR_CODE); + } else { + response = postQuery(sqlQuery); + assertTrue(response.get("exceptions").get(0).get("message").toString().startsWith("\"QueryExecutionError")); + } // invalid argument - sqlQuery = "SELECT toBase64('hello!') FROM myTable"; - response = postQuery(sqlQuery); - assertTrue(response.get("exceptions").get(0).get("message").toString().contains("SqlCompilationException")); + sqlQuery = "SELECT toBase64('hello!') FROM mytable"; + if (useMultiStageQueryEngine) { + testQueryError(sqlQuery, QueryException.QUERY_PLANNING_ERROR_CODE); + } else { + response = postQuery(sqlQuery); + assertTrue(response.get("exceptions").get(0).get("message").toString().contains("SqlCompilationException")); + } // invalid argument - sqlQuery = "SELECT fromBase64('hello!') FROM myTable"; - response = postQuery(sqlQuery); - assertTrue(response.get("exceptions").get(0).get("message").toString().contains("IllegalArgumentException")); + sqlQuery = "SELECT fromBase64('hello!') FROM mytable"; + if (useMultiStageQueryEngine) { + testQueryError(sqlQuery, QueryException.QUERY_PLANNING_ERROR_CODE); + } else { + response = postQuery(sqlQuery); + assertTrue(response.get("exceptions").get(0).get("message").toString().contains("IllegalArgumentException")); + } // string literal used in a filter - sqlQuery = "SELECT * FROM myTable WHERE fromUtf8(fromBase64('aGVsbG8h')) != Carrier AND " + sqlQuery = "SELECT * FROM mytable WHERE fromUtf8(fromBase64('aGVsbG8h')) != Carrier AND " + "toBase64(toUtf8('hello!')) != Carrier LIMIT 10"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); @@ -884,21 +913,21 @@ public void testBase64Func() assertEquals(rows.size(), 10); // non-string literal used in a filter - sqlQuery = "SELECT * FROM myTable WHERE fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) != Carrier LIMIT 10"; + sqlQuery = "SELECT * FROM mytable WHERE fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) != Carrier LIMIT 10"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); rows = resultTable.get("rows"); assertEquals(rows.size(), 10); // string identifier used in a filter - sqlQuery = "SELECT * FROM myTable WHERE fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))) = Carrier LIMIT 10"; + sqlQuery = "SELECT * FROM mytable WHERE fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))) = Carrier LIMIT 10"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); rows = resultTable.get("rows"); assertEquals(rows.size(), 10); // non-string identifier used in a filter - sqlQuery = "SELECT fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))), AirlineID FROM myTable WHERE " + sqlQuery = "SELECT fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))), AirlineID FROM mytable WHERE " + "fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) = AirlineID LIMIT 10"; response = postQuery(sqlQuery); resultTable = response.get("resultTable"); @@ -909,7 +938,7 @@ public void testBase64Func() // string identifier used in group by order by sqlQuery = "SELECT Carrier as originalCol, toBase64(toUtf8(Carrier)) as encoded, " - + "fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))) as decoded FROM myTable " + + "fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))) as decoded FROM mytable " + "GROUP BY Carrier, toBase64(toUtf8(Carrier)), fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))) " + "ORDER BY toBase64(toUtf8(Carrier)) LIMIT 10"; response = postQuery(sqlQuery); @@ -929,7 +958,7 @@ public void testBase64Func() // non-string identifier used in group by order by sqlQuery = "SELECT AirlineID as originalCol, toBase64(toUtf8(AirlineID)) as encoded, " - + "fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) as decoded FROM myTable " + + "fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) as decoded FROM mytable " + "GROUP BY AirlineID, toBase64(toUtf8(AirlineID)), fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) " + "ORDER BY fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) LIMIT 10"; response = postQuery(sqlQuery); @@ -949,7 +978,7 @@ public void testBase64Func() } @Test - public void testLiteralOnlyFunc() + public void testLiteralOnlyFuncV1() throws Exception { long queryStartTimeMs = System.currentTimeMillis(); String sqlQuery = @@ -1011,9 +1040,75 @@ public void testLiteralOnlyFunc() assertEquals(results.get(10).asText(), "hello!"); } - @Test(dependsOnMethods = "testBloomFilterTriggering") - public void testRangeIndexTriggering() + @Test + public void testLiteralOnlyFuncV2() throws Exception { + setUseMultiStageQueryEngine(true); + long queryStartTimeMs = System.currentTimeMillis(); + String sqlQuery = + "SELECT 1, now() as currentTs, ago('PT1H') as oneHourAgoTs, 'abc', toDateTime(now(), 'yyyy-MM-dd z') as " + + "today, now(), ago('PT1H'), encodeUrl('key1=value 1&key2=value@!$2&key3=value%3') as encodedUrl, " + + "decodeUrl('key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253') as decodedUrl, toBase64" + + "(toUtf8('hello!')) as toBase64, fromUtf8(fromBase64('aGVsbG8h')) as fromBase64"; + JsonNode response = postQuery(sqlQuery); + long queryEndTimeMs = System.currentTimeMillis(); + + JsonNode resultTable = response.get("resultTable"); + JsonNode dataSchema = resultTable.get("dataSchema"); + JsonNode columnNames = dataSchema.get("columnNames"); + assertEquals(columnNames.get(0).asText(), "EXPR$0"); + assertEquals(columnNames.get(1).asText(), "currentTs"); + assertEquals(columnNames.get(2).asText(), "oneHourAgoTs"); + assertEquals(columnNames.get(3).asText(), "EXPR$3"); + assertEquals(columnNames.get(4).asText(), "today"); + String nowColumnName = columnNames.get(5).asText(); + String oneHourAgoColumnName = columnNames.get(6).asText(); + assertEquals(columnNames.get(7).asText(), "encodedUrl"); + assertEquals(columnNames.get(8).asText(), "decodedUrl"); + assertEquals(columnNames.get(9).asText(), "toBase64"); + assertEquals(columnNames.get(10).asText(), "fromBase64"); + + JsonNode columnDataTypes = dataSchema.get("columnDataTypes"); + assertEquals(columnDataTypes.get(0).asText(), "INT"); + assertEquals(columnDataTypes.get(1).asText(), "LONG"); + assertEquals(columnDataTypes.get(2).asText(), "LONG"); + assertEquals(columnDataTypes.get(3).asText(), "STRING"); + assertEquals(columnDataTypes.get(4).asText(), "STRING"); + assertEquals(columnDataTypes.get(5).asText(), "LONG"); + assertEquals(columnDataTypes.get(6).asText(), "LONG"); + assertEquals(columnDataTypes.get(7).asText(), "STRING"); + assertEquals(columnDataTypes.get(8).asText(), "STRING"); + assertEquals(columnDataTypes.get(9).asText(), "STRING"); + assertEquals(columnDataTypes.get(10).asText(), "STRING"); + + JsonNode results = resultTable.get("rows").get(0); + assertEquals(results.get(0).asInt(), 1); + long nowResult = results.get(1).asLong(); + assertTrue(nowResult >= queryStartTimeMs); + assertTrue(nowResult <= queryEndTimeMs); + long oneHourAgoResult = results.get(2).asLong(); + assertTrue(oneHourAgoResult >= queryStartTimeMs - TimeUnit.HOURS.toMillis(1)); + assertTrue(oneHourAgoResult <= queryEndTimeMs - TimeUnit.HOURS.toMillis(1)); + assertEquals(results.get(3).asText(), "abc"); + String queryStartTimeDay = Instant.ofEpochMilli(queryStartTimeMs).atZone(ZoneId.of("UTC")) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd z")); + String queryEndTimeDay = Instant.ofEpochMilli(queryEndTimeMs).atZone(ZoneId.of("UTC")) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd z")); + String dateTimeResult = results.get(4).asText(); + assertTrue(dateTimeResult.equals(queryStartTimeDay) || dateTimeResult.equals(queryEndTimeDay)); + // In V2 column names and values are not related +// assertEquals(results.get(5).asText(), nowColumnName); +// assertEquals(results.get(6).asText(), oneHourAgoColumnName); + assertEquals(results.get(7).asText(), "key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253"); + assertEquals(results.get(8).asText(), "key1=value 1&key2=value@!$2&key3=value%3"); + assertEquals(results.get(9).asText(), "aGVsbG8h"); + assertEquals(results.get(10).asText(), "hello!"); + } + + @Test(dependsOnMethods = "testBloomFilterTriggering", dataProvider = "useBothQueryEngines") + public void testRangeIndexTriggering(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); long numTotalDocs = getCountStarResult(); assertEquals(postQuery(TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), numTotalDocs); @@ -1095,9 +1190,10 @@ public void testBloomFilterTriggering() /** * Check if server returns error response quickly without timing out Broker. */ - @Test - public void testServerErrorWithBrokerTimeout() + @Test(dataProvider = "useBothQueryEngines") + public void testServerErrorWithBrokerTimeout(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); long startTimeMs = System.currentTimeMillis(); // The query below will fail execution due to JSON_MATCH on column without json index JsonNode queryResponse = postQuery("SELECT count(*) FROM mytable WHERE JSON_MATCH(Dest, '$=123')"); @@ -1275,9 +1371,11 @@ public void testStarTreeTriggering() *
  • "NewAddedDerivedMVStringDimension", DATE_TIME, STRING, multi-value
    • * */ - @Test(dependsOnMethods = "testAggregateMetadataAPI") - public void testDefaultColumns() + @Test(dependsOnMethods = "testAggregateMetadataAPI", dataProvider = "useBothQueryEngines") + public void testDefaultColumns(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); long numTotalDocs = getCountStarResult(); reloadWithExtraColumns(); @@ -1305,7 +1403,7 @@ public void testDefaultColumns() public void testDisableGroovyQueryTableConfigOverride() throws Exception { String groovyQuery = "SELECT GROOVY('{\"returnType\":\"STRING\",\"isSingleValue\":true}', " - + "'arg0 + arg1', FlightNum, Origin) FROM myTable"; + + "'arg0 + arg1', FlightNum, Origin) FROM mytable"; TableConfig tableConfig = getOfflineTableConfig(); tableConfig.setQueryConfig(new QueryConfig(null, false, null, null)); updateTableConfig(tableConfig); @@ -1618,24 +1716,24 @@ private void testExpressionOverride() }, 60_000L, "Failed to remove expression override"); } - @Test - @Override - public void testBrokerResponseMetadata() + @Test(dataProvider = "useBothQueryEngines") + public void testBrokerResponseMetadata(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); super.testBrokerResponseMetadata(); } - @Test - public void testInBuiltVirtualColumns() + @Test(dataProvider = "useBothQueryEngines") + public void testInBuiltVirtualColumns(boolean useMultiStageQueryEngine) throws Exception { - String query = "SELECT $docId, $HOSTNAME, $segmentname FROM mytable"; + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SELECT $docId, $hostName, $segmentName FROM mytable"; JsonNode response = postQuery(query); JsonNode resultTable = response.get("resultTable"); JsonNode dataSchema = resultTable.get("dataSchema"); assertEquals(dataSchema.get("columnNames").toString(), "[\"$docId\",\"$hostName\",\"$segmentName\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"INT\",\"STRING\",\"STRING\"]"); JsonNode rows = resultTable.get("rows"); - assertEquals(rows.size(), 10); String expectedHostName = NetUtils.getHostnameOrAddress(); String expectedSegmentNamePrefix = "mytable_"; for (int i = 0; i < 10; i++) { @@ -1646,19 +1744,19 @@ public void testInBuiltVirtualColumns() } } - @Test - public void testGroupByUDF() + @Test(dataProvider = "useBothQueryEngines") + public void testGroupByUDF(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); String query = "SELECT timeConvert(DaysSinceEpoch,'DAYS','SECONDS'), COUNT(*) FROM mytable " + "GROUP BY timeConvert(DaysSinceEpoch,'DAYS','SECONDS') ORDER BY COUNT(*) DESC"; JsonNode response = postQuery(query); JsonNode resultTable = response.get("resultTable"); JsonNode dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), - "[\"timeconvert(DaysSinceEpoch,'DAYS','SECONDS')\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"LONG\",\"LONG\"]"); JsonNode rows = resultTable.get("rows"); - assertEquals(rows.size(), 10); + assertFalse(rows.isEmpty()); JsonNode row = rows.get(0); assertEquals(row.size(), 2); assertEquals(row.get(0).asLong(), 16138 * 24 * 3600); @@ -1669,26 +1767,27 @@ public void testGroupByUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), - "[\"datetimeconvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH','1:HOURS')\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"LONG\",\"LONG\"]"); rows = resultTable.get("rows"); - assertEquals(rows.size(), 10); + assertFalse(rows.isEmpty()); row = rows.get(0); assertEquals(row.size(), 2); assertEquals(row.get(0).asLong(), 16138 * 24); assertEquals(row.get(1).asLong(), 605); - query = "SELECT add(DaysSinceEpoch,DaysSinceEpoch,15), COUNT(*) FROM mytable " - + "GROUP BY add(DaysSinceEpoch,DaysSinceEpoch,15) ORDER BY COUNT(*) DESC"; + if (useMultiStageQueryEngine) { + query = "SELECT add(DaysSinceEpoch,add(DaysSinceEpoch,15)), COUNT(*) FROM mytable " + + "GROUP BY add(DaysSinceEpoch,add(DaysSinceEpoch,15)) ORDER BY COUNT(*) DESC"; + } else { + query = "SELECT add(DaysSinceEpoch,DaysSinceEpoch,15), COUNT(*) FROM mytable " + + "GROUP BY add(DaysSinceEpoch,DaysSinceEpoch,15) ORDER BY COUNT(*) DESC"; + } response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), - "[\"add(DaysSinceEpoch,DaysSinceEpoch,'15')\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"DOUBLE\",\"LONG\"]"); rows = resultTable.get("rows"); - assertEquals(rows.size(), 10); + assertFalse(rows.isEmpty()); row = rows.get(0); assertEquals(row.size(), 2); assertEquals(row.get(0).asDouble(), 16138.0 + 16138 + 15); @@ -1699,24 +1798,27 @@ public void testGroupByUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), "[\"sub(DaysSinceEpoch,'25')\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"DOUBLE\",\"LONG\"]"); rows = resultTable.get("rows"); - assertEquals(rows.size(), 10); + assertFalse(rows.isEmpty()); row = rows.get(0); assertEquals(row.size(), 2); assertEquals(row.get(0).asDouble(), 16138.0 - 25); assertEquals(row.get(1).asLong(), 605); - query = "SELECT mult(DaysSinceEpoch,24,3600), COUNT(*) FROM mytable " - + "GROUP BY mult(DaysSinceEpoch,24,3600) ORDER BY COUNT(*) DESC"; + if (useMultiStageQueryEngine) { + query = "SELECT mult(DaysSinceEpoch,mult(24,3600)), COUNT(*) FROM mytable " + + "GROUP BY mult(DaysSinceEpoch,mult(24,3600)) ORDER BY COUNT(*) DESC"; + } else { + query = "SELECT mult(DaysSinceEpoch,24,3600), COUNT(*) FROM mytable " + + "GROUP BY mult(DaysSinceEpoch,24,3600) ORDER BY COUNT(*) DESC"; + } response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), "[\"mult(DaysSinceEpoch,'24','3600')\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"DOUBLE\",\"LONG\"]"); rows = resultTable.get("rows"); - assertEquals(rows.size(), 10); + assertFalse(rows.isEmpty()); row = rows.get(0); assertEquals(row.size(), 2); assertEquals(row.get(0).asDouble(), 16138.0 * 24 * 3600); @@ -1727,10 +1829,9 @@ public void testGroupByUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), "[\"div(DaysSinceEpoch,'2')\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"DOUBLE\",\"LONG\"]"); rows = resultTable.get("rows"); - assertEquals(rows.size(), 10); + assertFalse(rows.isEmpty()); row = rows.get(0); assertEquals(row.size(), 2); assertEquals(row.get(0).asDouble(), 16138.0 / 2); @@ -1741,10 +1842,9 @@ public void testGroupByUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), "[\"arraylength(DivAirports)\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"INT\",\"LONG\"]"); rows = resultTable.get("rows"); - assertEquals(rows.size(), 1); + assertFalse(rows.isEmpty()); row = rows.get(0); assertEquals(row.size(), 2); assertEquals(row.get(0).asInt(), 5); @@ -1755,8 +1855,6 @@ public void testGroupByUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), - "[\"arraylength(valuein(DivAirports,'DFW','ORD'))\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"INT\",\"LONG\"]"); rows = resultTable.get("rows"); assertEquals(rows.size(), 3); @@ -1778,7 +1876,6 @@ public void testGroupByUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), "[\"valuein(DivAirports,'DFW','ORD')\",\"count(*)\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"STRING\",\"LONG\"]"); rows = resultTable.get("rows"); assertEquals(rows.size(), 2); @@ -1793,7 +1890,7 @@ public void testGroupByUDF() } @Test - public void testAggregationUDF() + public void testAggregationUDFV1() throws Exception { String query = "SELECT MAX(timeConvert(DaysSinceEpoch,'DAYS','SECONDS')) FROM mytable"; JsonNode response = postQuery(query); @@ -1821,14 +1918,40 @@ public void testAggregationUDF() } @Test - public void testSelectionUDF() + public void testAggregationUDFV2() throws Exception { - String query = "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable"; + setUseMultiStageQueryEngine(true); + String query = "SELECT MAX(timeConvert(DaysSinceEpoch,'DAYS','SECONDS')) FROM mytable"; + JsonNode response = postQuery(query); + JsonNode resultTable = response.get("resultTable"); + JsonNode dataSchema = resultTable.get("dataSchema"); + assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"LONG\"]"); + JsonNode rows = resultTable.get("rows"); + assertEquals(rows.size(), 1); + JsonNode row = rows.get(0); + assertEquals(row.size(), 1); + assertEquals(row.get(0).asDouble(), 16435.0 * 24 * 3600); + + query = "SELECT MIN(div(DaysSinceEpoch,2)) FROM mytable"; + response = postQuery(query); + resultTable = response.get("resultTable"); + dataSchema = resultTable.get("dataSchema"); + assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"DOUBLE\"]"); + rows = resultTable.get("rows"); + assertEquals(rows.size(), 1); + row = rows.get(0); + assertEquals(row.size(), 1); + assertEquals(row.get(0).asDouble(), 16071.0 / 2); + } + + @Test(dataProvider = "useBothQueryEngines") + public void testSelectionUDF(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable limit 10"; JsonNode response = postQuery(query); JsonNode resultTable = response.get("resultTable"); JsonNode dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), - "[\"DaysSinceEpoch\",\"timeconvert(DaysSinceEpoch,'DAYS','SECONDS')\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"INT\",\"LONG\"]"); JsonNode rows = response.get("resultTable").get("rows"); assertEquals(rows.size(), 10); @@ -1843,8 +1966,6 @@ public void testSelectionUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), - "[\"DaysSinceEpoch\",\"timeconvert(DaysSinceEpoch,'DAYS','SECONDS')\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"INT\",\"LONG\"]"); rows = response.get("resultTable").get("rows"); assertEquals(rows.size(), 10000); @@ -1862,8 +1983,6 @@ public void testSelectionUDF() response = postQuery(query); resultTable = response.get("resultTable"); dataSchema = resultTable.get("dataSchema"); - assertEquals(dataSchema.get("columnNames").toString(), - "[\"DaysSinceEpoch\",\"timeconvert(DaysSinceEpoch,'DAYS','SECONDS')\"]"); assertEquals(dataSchema.get("columnDataTypes").toString(), "[\"INT\",\"LONG\"]"); rows = response.get("resultTable").get("rows"); assertEquals(rows.size(), 10000); @@ -1877,9 +1996,10 @@ public void testSelectionUDF() } } - @Test - public void testFilterUDF() + @Test(dataProvider = "useBothQueryEngines") + public void testFilterUDF(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); int daysSinceEpoch = 16138; long secondsSinceEpoch = 16138 * 24 * 60 * 60; @@ -1912,22 +2032,23 @@ public void testFilterUDF() assertEquals(postQuery(query).get("resultTable").get("rows").get(0).get(0).asLong(), expectedResult); } - @Test - public void testCaseStatementInSelection() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseStatementInSelection(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); List origins = Arrays.asList("ATL", "ORD", "DFW", "DEN", "LAX", "IAH", "SFO", "PHX", "LAS", "EWR", "MCO", "BOS", "SLC", "SEA", "MSP", "CLT", "LGA", "DTW", "JFK", "BWI"); StringBuilder caseStatementBuilder = new StringBuilder("CASE "); for (int i = 0; i < origins.size(); i++) { - // WHEN origin = 'ATL' THEN 1 - // WHEN origin = 'ORD' THEN 2 - // WHEN origin = 'DFW' THEN 3 + // WHEN Origin = 'ATL' THEN 1 + // WHEN Origin = 'ORD' THEN 2 + // WHEN Origin = 'DFW' THEN 3 // .... - caseStatementBuilder.append(String.format("WHEN origin = '%s' THEN %d ", origins.get(i), i + 1)); + caseStatementBuilder.append(String.format("WHEN Origin = '%s' THEN %d ", origins.get(i), i + 1)); } caseStatementBuilder.append("ELSE 0 END"); - String sqlQuery = "SELECT origin, " + caseStatementBuilder + " AS origin_code FROM mytable LIMIT 1000"; + String sqlQuery = "SELECT Origin, " + caseStatementBuilder + " AS origin_code FROM mytable LIMIT 1000"; JsonNode response = postQuery(sqlQuery); JsonNode rows = response.get("resultTable").get("rows"); assertTrue(response.get("exceptions").isEmpty()); @@ -1942,9 +2063,10 @@ public void testCaseStatementInSelection() } } - @Test - public void testCaseStatementInSelectionWithTransformFunctionInThen() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseStatementInSelectionWithTransformFunctionInThen(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String sqlQuery = "SELECT ArrDelay, CASE WHEN ArrDelay > 0 THEN ArrDelay WHEN ArrDelay < 0 THEN ArrDelay * -1 ELSE 0 END AS " + "ArrTimeDiff FROM mytable LIMIT 1000"; @@ -1988,11 +2110,12 @@ public void testCaseStatementWithLogicalTransformFunction(boolean useMultiStageQ } } - @Test - public void testCaseStatementWithInAggregation() + @Test(dataProvider = "useBothQueryEngines") + public void testCaseStatementWithInAggregation(boolean useMultiStageQueryEngine) throws Exception { - testCountVsCaseQuery("origin = 'ATL'"); - testCountVsCaseQuery("origin <> 'ATL'"); + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + testCountVsCaseQuery("Origin = 'ATL'"); + testCountVsCaseQuery("Origin <> 'ATL'"); testCountVsCaseQuery("DaysSinceEpoch > 16312"); testCountVsCaseQuery("DaysSinceEpoch >= 16312"); @@ -2013,9 +2136,10 @@ private void testCountVsCaseQuery(String predicate) assertEquals(caseSum, countValue); } - @Test - public void testFilterWithInvertedIndexUDF() + @Test(dataProvider = "useBothQueryEngines") + public void testFilterWithInvertedIndexUDF(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); int daysSinceEpoch = 16138; long secondsSinceEpoch = 16138 * 24 * 60 * 60; @@ -2039,7 +2163,7 @@ public void testFilterWithInvertedIndexUDF() } @Test - public void testQueryWithRepeatedColumns() + public void testQueryWithRepeatedColumnsV1() throws Exception { //test repeated columns in selection query String query = "SELECT ArrTime, ArrTime FROM mytable WHERE DaysSinceEpoch <= 16312 AND Carrier = 'DL'"; @@ -2059,9 +2183,34 @@ public void testQueryWithRepeatedColumns() testQuery(query); } + // these tests actually checks a calcite limitation. + // Once it is fixed in calcite, we should merge this tests with testQueryRepetedColumnsV1 @Test - public void testQueryWithOrderby() + public void testQueryWithRepeatedColumnsV2() + throws Exception { + setUseMultiStageQueryEngine(true); + //test repeated columns in selection query + String query = "SELECT ArrTime, ArrTime FROM mytable WHERE DaysSinceEpoch <= 16312 AND Carrier = 'DL'"; + testQuery(query); + + //test repeated columns in selection query with order by + query = "SELECT ArrTime, ArrTime FROM mytable WHERE DaysSinceEpoch <= 16312 AND Carrier = 'DL' order by ArrTime"; + testQueryError(query, QueryException.QUERY_PLANNING_ERROR_CODE); + + //test repeated columns in agg query + query = "SELECT COUNT(*), COUNT(*) FROM mytable WHERE DaysSinceEpoch <= 16312 AND Carrier = 'DL'"; + testQuery(query); + + //test repeated columns in agg group by query + query = "SELECT ArrTime, ArrTime, COUNT(*), COUNT(*) FROM mytable WHERE DaysSinceEpoch <= 16312 AND Carrier = 'DL' " + + "GROUP BY ArrTime, ArrTime"; + testQueryError(query, QueryException.QUERY_PLANNING_ERROR_CODE); + } + + @Test(dataProvider = "useBothQueryEngines") + public void testQueryWithOrderby(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); //test repeated columns in selection query String query = "SELECT ArrTime, Carrier, DaysSinceEpoch FROM mytable ORDER BY DaysSinceEpoch DESC"; testQuery(query); @@ -2075,9 +2224,10 @@ public void testQueryWithOrderby() testQuery(query); } - @Test - public void testQueryWithAlias() + @Test(dataProvider = "useBothQueryEngines") + public void testQueryWithAlias(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); { //test same alias name with column name String query = "SELECT ArrTime AS ArrTime, Carrier AS Carrier, DaysSinceEpoch AS DaysSinceEpoch FROM mytable " @@ -2223,9 +2373,10 @@ private void testInstanceDecommission() assertFalse(_propertyStore.exists(configPath, 0)); } - @Test - public void testDistinctQuery() + @Test(dataProvider = "useBothQueryEngines") + public void testDistinctQuery(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // by default 10 rows will be returned, so use high limit String pinotQuery = "SELECT DISTINCT Carrier FROM mytable LIMIT 1000000"; String h2Query = "SELECT DISTINCT Carrier FROM mytable"; @@ -2244,9 +2395,10 @@ public void testDistinctQuery() testQuery(pinotQuery, h2Query); } - @Test - public void testNonAggregationGroupByQuery() + @Test(dataProvider = "useBothQueryEngines") + public void testNonAggregationGroupByQuery(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // by default 10 rows will be returned, so use high limit String pinotQuery = "SELECT Carrier FROM mytable GROUP BY Carrier LIMIT 1000000"; String h2Query = "SELECT Carrier FROM mytable GROUP BY Carrier"; @@ -2276,9 +2428,9 @@ public void testNonAggregationGroupByQuery() testQuery(pinotQuery, h2Query); } - @Test - public void testCaseInsensitivity() + public void testCaseInsensitivityV1(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); int daysSinceEpoch = 16138; int hoursSinceEpoch = 16138 * 24; int secondsSinceEpoch = 16138 * 24 * 60 * 60; @@ -2305,6 +2457,35 @@ public void testCaseInsensitivity() } } + @Test + public void testCaseSensitivityV2() + throws Exception { + setUseMultiStageQueryEngine(true); + int daysSinceEpoch = 16138; + int hoursSinceEpoch = 16138 * 24; + int secondsSinceEpoch = 16138 * 24 * 60 * 60; + List baseQueries = Arrays.asList("SELECT * FROM mytable", + "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable", + "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable order by DaysSinceEpoch " + + "limit 10000", + "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable order by timeConvert" + + "(DaysSinceEpoch,'DAYS','SECONDS') DESC limit 10000", + "SELECT count(*) FROM mytable WHERE DaysSinceEpoch = " + daysSinceEpoch, + "SELECT count(*) FROM mytable WHERE timeConvert(DaysSinceEpoch,'DAYS','HOURS') = " + hoursSinceEpoch, + "SELECT count(*) FROM mytable WHERE timeConvert(DaysSinceEpoch,'DAYS','SECONDS') = " + secondsSinceEpoch, + "SELECT MAX(timeConvert(DaysSinceEpoch,'DAYS','SECONDS')) FROM mytable", + "SELECT COUNT(*) FROM mytable GROUP BY dateTimeConvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH'," + + "'1:HOURS')"); + List queries = new ArrayList<>(); + baseQueries.forEach(q -> queries.add(q.replace("mytable", "MYTABLE").replace("DaysSinceEpoch", "DAYSSinceEpOch"))); + baseQueries.forEach( + q -> queries.add(q.replace("mytable", "MYDB.MYTABLE").replace("DaysSinceEpoch", "DAYSSinceEpOch"))); + + for (String query : queries) { + testQueryError(query, QueryException.QUERY_PLANNING_ERROR_CODE); + } + } + @Test public void testColumnNameContainsTableName() throws Exception { @@ -2334,7 +2515,7 @@ public void testColumnNameContainsTableName() } @Test - public void testCaseInsensitivityWithColumnNameContainsTableName() + public void testCaseInsensitivityWithColumnNameContainsTableNameV1() throws Exception { int daysSinceEpoch = 16138; int hoursSinceEpoch = 16138 * 24; @@ -2365,7 +2546,38 @@ public void testCaseInsensitivityWithColumnNameContainsTableName() } @Test - public void testQuerySourceWithDatabaseName() + public void testCaseSensitivityWithColumnNameContainsTableNameV2() + throws Exception { + setUseMultiStageQueryEngine(true); + int daysSinceEpoch = 16138; + int hoursSinceEpoch = 16138 * 24; + int secondsSinceEpoch = 16138 * 24 * 60 * 60; + List baseQueries = Arrays.asList("SELECT * FROM mytable", + "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable", + "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable order by DaysSinceEpoch " + + "limit 10000", + "SELECT DaysSinceEpoch, timeConvert(DaysSinceEpoch,'DAYS','SECONDS') FROM mytable order by timeConvert" + + "(DaysSinceEpoch,'DAYS','SECONDS') DESC limit 10000", + "SELECT count(*) FROM mytable WHERE DaysSinceEpoch = " + daysSinceEpoch, + "SELECT count(*) FROM mytable WHERE timeConvert(DaysSinceEpoch,'DAYS','HOURS') = " + hoursSinceEpoch, + "SELECT count(*) FROM mytable WHERE timeConvert(DaysSinceEpoch,'DAYS','SECONDS') = " + secondsSinceEpoch, + "SELECT MAX(timeConvert(DaysSinceEpoch,'DAYS','SECONDS')) FROM mytable", + "SELECT COUNT(*) FROM mytable GROUP BY dateTimeConvert(DaysSinceEpoch,'1:DAYS:EPOCH','1:HOURS:EPOCH'," + + "'1:HOURS')"); + List queries = new ArrayList<>(); + baseQueries.forEach( + q -> queries.add(q.replace("mytable", "MYTABLE").replace("DaysSinceEpoch", "MYTABLE.DAYSSinceEpOch"))); + // something like "SELECT MYDB.MYTABLE.DAYSSinceEpOch from MYDB.MYTABLE where MYDB.MYTABLE.DAYSSinceEpOch = 16138" + baseQueries.forEach( + q -> queries.add(q.replace("mytable", "MYDB.MYTABLE").replace("DaysSinceEpoch", "MYTABLE.DAYSSinceEpOch"))); + + for (String query : queries) { + testQueryError(query, QueryException.QUERY_PLANNING_ERROR_CODE); + } + } + + @Test + public void testQuerySourceWithDatabaseNameV1() throws Exception { // by default 10 rows will be returned, so use high limit String pinotQuery = "SELECT DISTINCT(Carrier) FROM mytable LIMIT 1000000"; @@ -2375,9 +2587,28 @@ public void testQuerySourceWithDatabaseName() testQuery(pinotQuery, h2Query); } + @Test - public void testDistinctCountHll() + public void testQuerySourceWithDatabaseNameV2() throws Exception { + setUseMultiStageQueryEngine(true); + // by default 10 rows will be returned, so use high limit + String pinotQuery = "SELECT DISTINCT(Carrier) FROM mytable LIMIT 1000000"; + String h2Query = "SELECT DISTINCT Carrier FROM mytable"; + testQuery(pinotQuery, h2Query); + + pinotQuery = "SELECT DISTINCT Carrier FROM db.mytable LIMIT 1000000"; + JsonNode response = postQuery(pinotQuery); + JsonNode exceptions = response.get("exceptions"); + assertFalse(exceptions.isEmpty(), "At least one exception was expected"); + JsonNode firstException = exceptions.get(0); + assertEquals(firstException.get("errorCode").asInt(), QueryException.QUERY_PLANNING_ERROR_CODE); + } + + @Test(dataProvider = "useBothQueryEngines") + public void testDistinctCountHll(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String query; // The Accurate value is 6538. @@ -2396,14 +2627,20 @@ public void testDistinctCountHll() assertEquals(postQuery(query).get("resultTable").get("rows").get(0).get(0).asLong(), expectedResults[i - 2]); } - // Default HLL is set as log2m=12 + // Default log2m for HLL is set to 12 in V1 and 8 in V2 + long expectedDefault; query = "SELECT distinctCountHLL(FlightNum) FROM mytable "; - assertEquals(postQuery(query).get("resultTable").get("rows").get(0).get(0).asLong(), expectedResults[10]); - assertEquals(postQuery(query).get("resultTable").get("rows").get(0).get(0).asLong(), expectedResults[10]); + if (useMultiStageQueryEngine) { + expectedDefault = expectedResults[6]; + } else { + expectedDefault = expectedResults[10]; + } + assertEquals(postQuery(query).get("resultTable").get("rows").get(0).get(0).asLong(), expectedDefault); + assertEquals(postQuery(query).get("resultTable").get("rows").get(0).get(0).asLong(), expectedDefault); } @Test - public void testAggregationFunctionsWithUnderscore() + public void testAggregationFunctionsWithUnderscoreV1() throws Exception { String query; @@ -2417,7 +2654,22 @@ public void testAggregationFunctionsWithUnderscore() } @Test - public void testExplainPlanQuery() + public void testAggregationFunctionsWithUnderscoreV2() + throws Exception { + setUseMultiStageQueryEngine(true); + String query; + + // The Accurate value is 6538. + query = "SELECT distinct_count(FlightNum) FROM mytable"; + assertEquals(postQuery(query).get("resultTable").get("rows").get(0).get(0).asInt(), 6538); + + // This is not supported in V2. + query = "SELECT c_o_u_n_t(FlightNum) FROM mytable"; + testQueryError(query, QueryException.QUERY_PLANNING_ERROR_CODE); + } + + @Test + public void testExplainPlanQueryV1() throws Exception { String query1 = "EXPLAIN PLAN FOR SELECT count(*) AS count, Carrier AS name FROM mytable GROUP BY name ORDER BY 1"; String response1 = postQuery(query1).get("resultTable").toString(); @@ -2444,35 +2696,77 @@ public void testExplainPlanQuery() + "[\"PLAN_START(numSegmentsForThisPlan:12)\",-1,-1],[\"ALL_SEGMENTS_PRUNED_ON_SERVER\",2,1]]}"); } - /** Test to make sure we are properly handling string comparisons in predicates. */ @Test - public void testStringComparisonInFilter() + public void testExplainPlanQueryV2() throws Exception { - // compare two string columns. - String query1 = "SELECT count(*) FROM mytable WHERE OriginState = DestState"; + setUseMultiStageQueryEngine(true); + String query1 = "EXPLAIN PLAN FOR SELECT count(*) AS count, Carrier AS name FROM mytable GROUP BY name ORDER BY 1"; String response1 = postQuery(query1).get("resultTable").toString(); - assertEquals(response1, - "{\"dataSchema\":{\"columnNames\":[\"count(*)\"],\"columnDataTypes\":[\"LONG\"]}," + "\"rows\":[[14011]]}"); - // compare string function with string column. - String query2 = "SELECT count(*) FROM mytable WHERE trim(OriginState) = DestState"; + // Replace string "docs:[0-9]+" with "docs:*" so that test doesn't fail when number of documents change. This is + // needed because both OfflineClusterIntegrationTest and MultiNodesOfflineClusterIntegrationTest run this test + // case with different number of documents in the segment. + response1 = response1.replaceAll("docs:[0-9]+", "docs:*"); + + assertEquals(response1, "{\"dataSchema\":{\"columnNames\":[\"SQL\",\"PLAN\"],\"columnDataTypes\":[\"STRING\"," + + "\"STRING\"]},\"rows\":[[\"EXPLAIN PLAN FOR SELECT count(*) AS count, Carrier AS name FROM mytable " + + "GROUP BY name ORDER BY 1\",\"Execution Plan\\n" + + "LogicalSort(sort0=[$0], dir0=[ASC], offset=[0])\\n" + + " PinotLogicalSortExchange(" + + "distribution=[hash], collation=[[0]], isSortOnSender=[false], isSortOnReceiver=[true])\\n" + + " LogicalSort(sort0=[$0], dir0=[ASC])\\n" + + " LogicalProject(count=[$1], name=[$0])\\n" + + " LogicalAggregate(group=[{0}], agg#0=[COUNT($1)])\\n" + + " PinotLogicalExchange(distribution=[hash[0]])\\n" + + " LogicalAggregate(group=[{17}], agg#0=[COUNT()])\\n" + + " LogicalTableScan(table=[[mytable]])\\n" + + "\"]]}"); + + // In the query below, FlightNum column has an inverted index and there is no data satisfying the predicate + // "FlightNum < 0". Hence, all segments are pruned out before query execution on the server side. + String query2 = "EXPLAIN PLAN FOR SELECT * FROM mytable WHERE FlightNum < 0"; String response2 = postQuery(query2).get("resultTable").toString(); - assertEquals(response2, - "{\"dataSchema\":{\"columnNames\":[\"count(*)\"],\"columnDataTypes\":[\"LONG\"]}," + "\"rows\":[[14011]]}"); + + Pattern pattern = Pattern.compile("\\{\"dataSchema\":\\{\"columnNames\":\\[\"SQL\",\"PLAN\"]," + + "\"columnDataTypes\":\\[\"STRING\",\"STRING\"]}," + + "\"rows\":\\[\\[\"EXPLAIN PLAN FOR SELECT \\* FROM mytable WHERE FlightNum < 0\"," + + "\"Execution Plan.." + + "LogicalProject\\(.*\\).." + + " LogicalFilter\\(condition=\\[<\\(.*, 0\\)]\\).." + + " LogicalTableScan\\(table=\\[\\[mytable]]\\)..\"" + + "]]}"); + boolean found = pattern.matcher(response2).find(); + assertTrue(found, "Pattern " + pattern + " not found in " + response2); + } + + /** Test to make sure we are properly handling string comparisons in predicates. */ + @Test(dataProvider = "useBothQueryEngines") + public void testStringComparisonInFilter(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + // compare two string columns. + JsonNode jsonNode = postQuery("SELECT count(*) FROM mytable WHERE OriginState = DestState"); + assertEquals(getType(jsonNode, 0), "LONG"); + assertEquals(getLongCellValue(jsonNode, 0, 0), 14011); + + // compare string function with string column. + jsonNode = postQuery("SELECT count(*) FROM mytable WHERE trim(OriginState) = DestState"); + assertEquals(getType(jsonNode, 0), "LONG"); + assertEquals(getLongCellValue(jsonNode, 0, 0), 14011); // compare string function with string function. - String query3 = "SELECT count(*) FROM mytable WHERE substr(OriginState, 0, 1) = substr(DestState, 0, 1)"; - String response3 = postQuery(query3).get("resultTable").toString(); - assertEquals(response3, - "{\"dataSchema\":{\"columnNames\":[\"count(*)\"],\"columnDataTypes\":[\"LONG\"]}," + "\"rows\":[[19755]]}"); + jsonNode = postQuery("SELECT count(*) FROM mytable WHERE substr(OriginState, 0, 1) = substr(DestState, 0, 1)"); + assertEquals(getType(jsonNode, 0), "LONG"); + assertEquals(getLongCellValue(jsonNode, 0, 0), 19755); } /** * Test queries that can be solved with {@link NonScanBasedAggregationOperator}. */ - @Test - public void testNonScanAggregationQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testNonScanAggregationQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); String tableName = getTableName(); // Test queries with COUNT, MIN, MAX, MIN_MAX_RANGE diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java index 7cc9a60fe69f..7c762fa4a48f 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterMemBasedServerQueryKillingTest.java @@ -211,9 +211,11 @@ protected TableConfig createOfflineTableConfig() { .build(); } - @Test - public void testDigestOOM() + @Test(dataProvider = "useBothQueryEngines") + public void testDigestOOM(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); JsonNode queryResponse = postQuery(OOM_QUERY); Assert.assertTrue(queryResponse.get("exceptions").toString().contains("\"errorCode\":" + QueryException.QUERY_CANCELLATION_ERROR_CODE)); @@ -221,17 +223,21 @@ public void testDigestOOM() Assert.assertTrue(queryResponse.get("exceptions").toString().contains("got killed because")); } - @Test - public void testDigestOOM2() + @Test(dataProvider = "useBothQueryEngines") + public void testDigestOOM2(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); JsonNode queryResponse = postQuery(OOM_QUERY_2); Assert.assertTrue(queryResponse.get("exceptions").toString().contains("QueryCancelledException")); Assert.assertTrue(queryResponse.get("exceptions").toString().contains("got killed because")); } - @Test - public void testDigestOOMMultipleQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testDigestOOMMultipleQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); AtomicReference queryResponse1 = new AtomicReference<>(); AtomicReference queryResponse2 = new AtomicReference<>(); AtomicReference queryResponse3 = new AtomicReference<>(); diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java index ce8a12a47852..35d8ea3b5eee 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterServerCPUTimeQueryKillingTest.java @@ -214,9 +214,11 @@ protected TableConfig createOfflineTableConfig() { .build(); } - @Test - public void testDigestTimeoutMultipleQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testDigestTimeoutMultipleQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + notSupportedInV2(); AtomicReference queryResponse1 = new AtomicReference<>(); AtomicReference queryResponse2 = new AtomicReference<>(); AtomicReference queryResponse3 = new AtomicReference<>(); diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java new file mode 100644 index 000000000000..97863881568f --- /dev/null +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineGRPCServerMultiStageIntegrationTest.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.integration.tests; + +import org.testng.annotations.BeforeTest; + + +public class OfflineGRPCServerMultiStageIntegrationTest extends OfflineGRPCServerIntegrationTest { + @BeforeTest + void enableMultiStage() { + setUseMultiStageQueryEngine(true); + } +} diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java index e36f3d6aac44..6f91ad4af061 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java @@ -155,18 +155,20 @@ private static StarTreeIndexConfig getStarTreeIndexConfig(List dimension return new StarTreeIndexConfig(dimensions, null, functionColumnPairs, maxLeafRecords); } - @Test - public void testGeneratedQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testGeneratedQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); for (int i = 0; i < NUM_QUERIES_TO_GENERATE; i += 2) { testStarQuery(_starTree1QueryGenerator.nextQuery()); testStarQuery(_starTree2QueryGenerator.nextQuery()); } } - @Test - public void testHardCodedQueries() + @Test(dataProvider = "useBothQueryEngines") + public void testHardCodedQueries(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // This query can test the case of one predicate matches all the child nodes but star-node cannot be used because // the predicate is included as remaining predicate from another branch String starQuery = "SELECT DepTimeBlk, COUNT(*) FROM mytable " diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TextIndicesTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TextIndicesTest.java index a8535ea798dc..9d963eab7537 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TextIndicesTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/TextIndicesTest.java @@ -178,9 +178,10 @@ public TableConfig createOfflineTableConfig() { .setQueryConfig(getQueryConfig()).setNullHandlingEnabled(getNullHandlingEnabled()).build(); } - @Test - public void testTextSearchCountQuery() + @Test(dataProvider = "useBothQueryEngines") + public void testTextSearchCountQuery(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // Keep posting queries until all records are consumed long previousResult = 0; while (getCurrentCountStarResult() < NUM_RECORDS) { @@ -201,9 +202,10 @@ public void testTextSearchCountQuery() }, 10_000L, "Failed to reach expected number of matching records"); } - @Test - public void testTextSearchCountQueryNative() + @Test(dataProvider = "useBothQueryEngines") + public void testTextSearchCountQueryNative(boolean useMultiStageQueryEngine) throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); // Keep posting queries until all records are consumed long previousResult = 0; while (getCurrentCountStarResult() < NUM_RECORDS) {