From 6e60d1a2f0a97f5afab72d37d0f606625e21e798 Mon Sep 17 00:00:00 2001 From: py023 Date: Tue, 24 Dec 2024 22:20:31 +0800 Subject: [PATCH] [test](regression-test) use unified trigger_and_wait_compaction method (#45761) Issue Number: close #45591 Related PR: #38836 Problem Summary: Previously many regression cases use their own private functions to trigger compaction, results in a bunch of error-prone copy-pasted code. This PR introduces a unified method to trigger and wait compaction finished. Hopefully new cases won't need to write their own compaction method in the future. Unsolved problem: Several cases(~20) have more dedicated actions to check compaction result, I didn't modify those cases in this PR, to avoid unexpected case behavior change. It's better to handle those cases in another PR, as it takes more times and this PR is already a huge change set. --- .../plugins/plugin_compaction.groovy | 156 ++++++++++++++++++ .../plugins/plugin_curl_requester.groovy | 8 +- .../compaction_width_array_column.groovy | 27 +-- .../compaction/test_base_compaction.groovy | 2 +- ...on_with_dup_key_max_file_size_limit.groovy | 60 ++----- .../test_compacation_with_delete.groovy | 32 +--- .../test_compaction_agg_keys.groovy | 4 +- ..._compaction_agg_keys_with_array_map.groovy | 37 +---- ...est_compaction_agg_keys_with_delete.groovy | 34 +--- .../test_compaction_cumu_delete.groovy | 55 +----- .../test_compaction_dup_keys.groovy | 37 +---- ...est_compaction_dup_keys_with_delete.groovy | 42 +---- .../test_compaction_uniq_keys.groovy | 4 +- ...test_compaction_uniq_keys_row_store.groovy | 35 +--- ...st_compaction_uniq_keys_with_delete.groovy | 34 +--- .../compaction/test_full_compaction.groovy | 47 +----- .../test_full_compaction_by_table_id.groovy | 51 +----- .../test_single_replica_compaction.groovy | 12 +- .../test_time_series_compaction_policy.groovy | 58 +------ .../test_vertical_compaction_agg_keys.groovy | 32 +--- .../test_vertical_compaction_agg_state.groovy | 32 +--- .../test_vertical_compaction_dup_keys.groovy | 36 +--- .../test_vertical_compaction_uniq_keys.groovy | 32 +--- ...test_partial_update_skip_compaction.groovy | 2 +- .../test_full_compaciton_run_status.groovy | 6 +- ...t_full_compaction_with_ordered_data.groovy | 70 +------- .../test_variant_bloom_filter.groovy | 34 +--- ...x_change_with_cumulative_compaction.groovy | 60 +------ ...t_index_change_with_full_compaction.groovy | 60 +------ .../test_index_compaction_dup_keys.groovy | 57 +------ .../test_index_compaction_unique_keys.groovy | 59 +------ ...ompaction_with_multi_index_segments.groovy | 68 +------- ...umulative_compaction_with_format_v2.groovy | 62 +------ .../test_mor_table_with_format_v2.groovy | 61 +------ .../test_mow_table_with_format_v2.groovy | 63 +------ ...e_replica_compaction_with_format_v2.groovy | 61 +------ .../test_dup_table_inverted_index.groovy | 73 ++------ .../inverted_index_p1/test_show_data.groovy | 46 +----- .../test_map_load_and_compaction.groovy | 28 +--- .../test_agg_keys_schema_change_datev2.groovy | 8 +- ...est_schema_change_varchar_to_datev2.groovy | 2 +- ...st_agg_keys_schema_change_decimalv2.groovy | 2 +- ...st_agg_keys_schema_change_decimalv3.groovy | 2 +- .../test_agg_keys_schema_change.groovy | 29 +--- .../test_agg_mv_schema_change.groovy | 30 +--- .../test_agg_rollup_schema_change.groovy | 28 +--- .../test_agg_vals_schema_change.groovy | 28 +--- .../test_dup_keys_schema_change.groovy | 2 +- .../test_dup_mv_schema_change.groovy | 2 +- .../test_dup_rollup_schema_change.groovy | 2 +- .../test_dup_vals_schema_change.groovy | 2 +- .../test_uniq_keys_schema_change.groovy | 2 +- .../test_uniq_mv_schema_change.groovy | 12 +- .../test_uniq_rollup_schema_change.groovy | 12 +- .../test_uniq_vals_schema_change.groovy | 2 +- .../test_varchar_schema_change.groovy | 8 +- .../variant_github_events_p2/load.groovy | 37 +---- .../compaction/test_compaction.groovy | 6 +- .../compaction_sparse_column.groovy | 14 +- .../test_compaction_extract_root.groovy | 42 +---- 60 files changed, 344 insertions(+), 1605 deletions(-) create mode 100644 regression-test/plugins/plugin_compaction.groovy diff --git a/regression-test/plugins/plugin_compaction.groovy b/regression-test/plugins/plugin_compaction.groovy new file mode 100644 index 00000000000000..385292eb1fe86d --- /dev/null +++ b/regression-test/plugins/plugin_compaction.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.Suite +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility; + +Suite.metaClass.be_get_compaction_status{ String ip, String port, String tablet_id /* param */-> + return curl("GET", String.format("http://%s:%s/api/compaction/run_status?tablet_id=%s", ip, port, tablet_id)) +} + +Suite.metaClass.be_get_overall_compaction_status{ String ip, String port /* param */-> + return curl("GET", String.format("http://%s:%s/api/compaction/run_status", ip, port)) +} + +Suite.metaClass.be_show_tablet_status{ String ip, String port, String tablet_id /* param */-> + return curl("GET", String.format("http://%s:%s/api/compaction/show?tablet_id=%s", ip, port, tablet_id)) +} + +Suite.metaClass._be_run_compaction = { String ip, String port, String tablet_id, String compact_type -> + return curl("POST", String.format("http://%s:%s/api/compaction/run?tablet_id=%s&compact_type=%s", + ip, port, tablet_id, compact_type)) +} + +Suite.metaClass.be_run_base_compaction = { String ip, String port, String tablet_id /* param */-> + return _be_run_compaction(ip, port, tablet_id, "base") +} + +logger.info("Added 'be_run_base_compaction' function to Suite") + +Suite.metaClass.be_run_cumulative_compaction = { String ip, String port, String tablet_id /* param */-> + return _be_run_compaction(ip, port, tablet_id, "cumulative") +} + +logger.info("Added 'be_run_cumulative_compaction' function to Suite") + +Suite.metaClass.be_run_full_compaction = { String ip, String port, String tablet_id /* param */-> + return _be_run_compaction(ip, port, tablet_id, "full") +} + +Suite.metaClass.be_run_full_compaction_by_table_id = { String ip, String port, String table_id /* param */-> + return curl("POST", String.format("http://%s:%s/api/compaction/run?table_id=%s&compact_type=full", ip, port, table_id)) +} + +logger.info("Added 'be_run_full_compaction' function to Suite") + +Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compaction_type, int timeout_seconds=300 -> + if (!(compaction_type in ["cumulative", "base", "full"])) { + throw new IllegalArgumentException("invalid compaction type: ${compaction_type}, supported types: cumulative, base, full") + } + + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + def tablets = sql_return_maparray """show tablets from ${table_name}""" + def exit_code, stdout, stderr + + def auto_compaction_disabled = sql("show create table ${table_name}")[0][1].contains('"disable_auto_compaction" = "true"') + def is_time_series_compaction = sql("show create table ${table_name}")[0][1].contains('"compaction_policy" = "time_series"') + + // 1. cache compaction status + def be_tablet_compaction_status = [:] + for (tablet in tablets) { + def be_host = backendId_to_backendIP["${tablet.BackendId}"] + def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"] + (exit_code, stdout, stderr) = be_show_tablet_status(be_host, be_port, tablet.TabletId) + assert exit_code == 0: "get tablet status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}" + + def tabletStatus = parseJson(stdout.trim()) + be_tablet_compaction_status.put("${be_host}-${tablet.TabletId}", tabletStatus) + } + // 2. trigger compaction + def triggered_tablets = [] + for (tablet in tablets) { + def be_host = backendId_to_backendIP["${tablet.BackendId}"] + def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"] + switch (compaction_type) { + case "cumulative": + (exit_code, stdout, stderr) = be_run_cumulative_compaction(be_host, be_port, tablet.TabletId) + break + case "base": + (exit_code, stdout, stderr) = be_run_base_compaction(be_host, be_port, tablet.TabletId) + break + case "full": + (exit_code, stdout, stderr) = be_run_full_compaction(be_host, be_port, tablet.TabletId) + break + } + assert exit_code == 0: "trigger compaction failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}" + def trigger_status = parseJson(stdout.trim()) + if (trigger_status.status.toLowerCase() != "success") { + if (trigger_status.status.toLowerCase() == "already_exist") { + triggered_tablets.add(tablet) // compaction already in queue, treat it as successfully triggered + } else if (!auto_compaction_disabled) { + // ignore the error if auto compaction enabled + } else { + throw new Exception("trigger compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${trigger_status.status}") + } + } else { + triggered_tablets.add(tablet) + } + } + + // 3. wait all compaction finished + def running = triggered_tablets.size() > 0 + Awaitility.await().atMost(timeout_seconds, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { + for (tablet in triggered_tablets) { + def be_host = backendId_to_backendIP["${tablet.BackendId}"] + def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"] + + (exit_code, stdout, stderr) = be_get_compaction_status(be_host, be_port, tablet.TabletId) + assert exit_code == 0: "get compaction status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}" + def compactionStatus = parseJson(stdout.trim()) + assert compactionStatus.status.toLowerCase() == "success": "compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${compactionStatus.status}" + // running is true means compaction is still running + running = compactionStatus.run_status + + if (!isCloudMode() && !is_time_series_compaction) { + (exit_code, stdout, stderr) = be_show_tablet_status(be_host, be_port, tablet.TabletId) + assert exit_code == 0: "get tablet status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}" + def tabletStatus = parseJson(stdout.trim()) + def oldStatus = be_tablet_compaction_status.get("${be_host}-${tablet.TabletId}") + // last compaction success time isn't updated, indicates compaction is not started(so we treat it as running and wait) + running = running || (oldStatus["last ${compaction_type} success time"] == tabletStatus["last ${compaction_type} success time"]) + if (running) { + logger.info("compaction is still running, be host: ${be_host}, tablet id: ${tablet.TabletId}, run status: ${compactionStatus.run_status}, old status: ${oldStatus}, new status: ${tabletStatus}") + return false + } + } else { + // 1. cloud mode doesn't show compaction success time in tablet status for the time being, + // 2. time series compaction sometimes doesn't update compaction success time + // so we solely check run_status for these two cases + if (running) { + logger.info("compaction is still running, be host: ${be_host}, tablet id: ${tablet.TabletId}") + return false + } + } + } + return true + }) + + assert !running: "wait compaction timeout, be host: ${be_host}" +} diff --git a/regression-test/plugins/plugin_curl_requester.groovy b/regression-test/plugins/plugin_curl_requester.groovy index c0c11bf594edd2..a7d29c03c75deb 100644 --- a/regression-test/plugins/plugin_curl_requester.groovy +++ b/regression-test/plugins/plugin_curl_requester.groovy @@ -27,6 +27,7 @@ import org.apache.http.conn.ConnectTimeoutException import org.apache.http.conn.HttpHostConnectException import org.codehaus.groovy.runtime.IOGroovyMethods + Suite.metaClass.http_client = { String method, String url /* param */ -> Suite suite = delegate as Suite if (method != "GET" && method != "POST") { @@ -35,7 +36,7 @@ Suite.metaClass.http_client = { String method, String url /* param */ -> if (!url || !(url =~ /^https?:\/\/.+/)) { throw new Exception("Invalid url: ${url}") } - + Integer timeout = 300 // seconds Integer maxRetries = 10 Integer retryCount = 0 @@ -119,7 +120,7 @@ Suite.metaClass.curl = { String method, String url, String body = null /* param if (url.isBlank()) { throw new Exception("invalid curl url, blank") } - + Integer timeout = 10; // 10 seconds; Integer maxRetries = 10; // Maximum number of retries Integer retryCount = 0; // Current retry count @@ -161,10 +162,8 @@ Suite.metaClass.curl = { String method, String url, String body = null /* param return [code, out, err] } - logger.info("Added 'curl' function to Suite") - Suite.metaClass.show_be_config = { String ip, String port /*param */ -> return curl("GET", String.format("http://%s:%s/api/show_config", ip, port)) } @@ -231,7 +230,6 @@ Suite.metaClass.update_all_be_config = { String key, Object value -> logger.info("Added 'update_all_be_config' function to Suite") - Suite.metaClass._be_report = { String ip, int port, String reportName -> def url = "http://${ip}:${port}/api/report/${reportName}" def result = Http.GET(url, true) diff --git a/regression-test/suites/compaction/compaction_width_array_column.groovy b/regression-test/suites/compaction/compaction_width_array_column.groovy index 4e3fed354c7d84..d433e81f555f26 100644 --- a/regression-test/suites/compaction/compaction_width_array_column.groovy +++ b/regression-test/suites/compaction/compaction_width_array_column.groovy @@ -88,32 +88,7 @@ suite('compaction_width_array_column', "p2") { while (isOverLap && tryCnt < 3) { isOverLap = false - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - assertEquals("success", compactJson.status.toLowerCase()) - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - + trigger_and_wait_compaction(tableName, "cumulative") for (def tablet in tablets) { String tablet_id = tablet.TabletId (code, out, err) = curl("GET", tablet.CompactionStatus) diff --git a/regression-test/suites/compaction/test_base_compaction.groovy b/regression-test/suites/compaction/test_base_compaction.groovy index d0c8074ca4c868..011b397a9fbdd9 100644 --- a/regression-test/suites/compaction/test_base_compaction.groovy +++ b/regression-test/suites/compaction/test_base_compaction.groovy @@ -27,7 +27,7 @@ suite("test_base_compaction") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) diff --git a/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy b/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy index bf2e6f5f2c66bb..fdaa83a4b3120d 100644 --- a/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy +++ b/regression-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy @@ -19,11 +19,11 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { def tableName = "test_base_compaction_with_dup_key_max_file_size_limit" - + // use customer table of tpch_sf100 def rows = 15000000 - def load_tpch_sf100_customer = { - def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString() + def load_tpch_sf100_customer = { + def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString() def rowCount = sql "select count(*) from ${tableName}" def s3BucketName = getS3BucketName() def s3WithProperties = """WITH S3 ( @@ -62,7 +62,7 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { } sleep(5000) } - } + } } try { String backend_id; @@ -72,7 +72,7 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -106,29 +106,6 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { } assertEquals(code, 0) return out - } - - def waitForCompaction = { be_host, be_http_port, tablet_id -> - // wait for all compactions done - boolean running = true - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) } sql """ DROP TABLE IF EXISTS ${tableName}; """ @@ -148,7 +125,7 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" = "true" ) - """ + """ def tablet = (sql_return_maparray """ show tablets from ${tableName}; """)[0] String tablet_id = tablet.TabletId @@ -164,10 +141,7 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { // [0-1] 0 // [2-2] 1G nooverlapping // cp: 3 - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - + trigger_and_wait_compaction(tableName, "cumulative") // rowsets: // [0-1] 0 @@ -180,21 +154,15 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { // [0-1] 0 // [2-2] 1G nooverlapping // [3-3] 1G nooverlapping - // cp: 4 - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - + // cp: 4 + trigger_and_wait_compaction(tableName, "cumulative") // The conditions for base compaction have been satisfied. // Since the size of first input rowset is 0, there is no file size limitation. (maybe fix it?) // rowsets: // [0-3] 2G nooverlapping - // cp: 4 - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "base", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - + // cp: 4 + trigger_and_wait_compaction(tableName, "base") // rowsets: // [0-3] 2G nooverlapping @@ -206,10 +174,7 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { // [0-3] 2G nooverlapping // [4-4] 1G nooverlapping // cp: 5 - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - + trigger_and_wait_compaction(tableName, "cumulative") // Due to the limit of config::base_compaction_dup_key_max_file_size_mbytes(1G), // can not do base compaction, return E-808 @@ -217,6 +182,7 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") { // [0-3] 2G nooverlapping // [4-4] 1G nooverlapping // cp: 5 + // WHAT: replace with plugin and handle fail? assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], "base", tablet_id).contains("E-808")); diff --git a/regression-test/suites/compaction/test_compacation_with_delete.groovy b/regression-test/suites/compaction/test_compacation_with_delete.groovy index e41f787c52276e..43aaaa5485af18 100644 --- a/regression-test/suites/compaction/test_compacation_with_delete.groovy +++ b/regression-test/suites/compaction/test_compacation_with_delete.groovy @@ -99,37 +99,7 @@ suite("test_compaction_with_delete") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) diff --git a/regression-test/suites/compaction/test_compaction_agg_keys.groovy b/regression-test/suites/compaction/test_compaction_agg_keys.groovy index 3681b2dcac0be1..0ecb3f7df8eca2 100644 --- a/regression-test/suites/compaction/test_compaction_agg_keys.groovy +++ b/regression-test/suites/compaction/test_compaction_agg_keys.groovy @@ -26,7 +26,7 @@ suite("test_compaction_agg_keys") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); backend_id = backendId_to_backendIP.keySet()[0] - + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -145,7 +145,7 @@ suite("test_compaction_agg_keys") { (code, out, err) = curl("GET", tablet.CompactionStatus) logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) - + assertEquals(code, 0) def tabletJson = parseJson(out.trim()) assert tabletJson.rowsets instanceof List diff --git a/regression-test/suites/compaction/test_compaction_agg_keys_with_array_map.groovy b/regression-test/suites/compaction/test_compaction_agg_keys_with_array_map.groovy index 1556d2f00a506f..bad84c832954b4 100644 --- a/regression-test/suites/compaction/test_compaction_agg_keys_with_array_map.groovy +++ b/regression-test/suites/compaction/test_compaction_agg_keys_with_array_map.groovy @@ -26,7 +26,7 @@ suite("test_compaction_agg_keys_with_array_map") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); backend_id = backendId_to_backendIP.keySet()[0] - + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -96,38 +96,7 @@ suite("test_compaction_agg_keys_with_array_map") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) @@ -137,7 +106,7 @@ suite("test_compaction_agg_keys_with_array_map") { (code, out, err) = curl("GET", tablet.CompactionStatus) logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) - + assertEquals(code, 0) def tabletJson = parseJson(out.trim()) assert tabletJson.rowsets instanceof List diff --git a/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy b/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy index 1610587602d575..99ea6077e46b17 100644 --- a/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy +++ b/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy @@ -115,44 +115,14 @@ suite("test_compaction_agg_keys_with_delete") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) int rowCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId - + (code, out, err) = curl("GET", tablet.CompactionStatus) logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) diff --git a/regression-test/suites/compaction/test_compaction_cumu_delete.groovy b/regression-test/suites/compaction/test_compaction_cumu_delete.groovy index 31f1331969e16e..1b5a022ae50230 100644 --- a/regression-test/suites/compaction/test_compaction_cumu_delete.groovy +++ b/regression-test/suites/compaction/test_compaction_cumu_delete.groovy @@ -66,53 +66,6 @@ suite("test_compaction_cumu_delete") { return } - - def triggerCompaction = { be_host, be_http_port, compact_type -> - // trigger compactions for all tablets in ${tableName} - String tablet_id = tablet[0] - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=${compact_type}") - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) - if (!disableAutoCompaction) { - return "Success" - } - assertEquals(code, 0) - return out - } - def waitForCompaction = { be_host, be_http_port -> - // wait for all compactions done - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet[0] - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - // insert 11 values for 11 version sql """ INSERT INTO ${tableName} VALUES (1, "a", 100); """ sql """ INSERT INTO ${tableName} VALUES (2, "a", 100); """ @@ -127,7 +80,7 @@ suite("test_compaction_cumu_delete") { // [0-1] [2-12] // write some key in version 13, delete it in version 14, write same key in version 15 // make sure the key in version 15 will not be deleted - assertTrue(triggerCompaction(backendId_to_backendIP[backend_id], backendId_to_backendHttpPort[backend_id], "base").contains("Success")); + trigger_and_wait_compaction(tableName, "base") sql """ INSERT INTO ${tableName} VALUES (4, "a", 100); """ qt_select_default """ SELECT * FROM ${tableName}; """ sql """ DELETE FROM ${tableName} WHERE id = 4; """ @@ -147,12 +100,10 @@ suite("test_compaction_cumu_delete") { sql """ INSERT INTO ${tableName} VALUES (7, "a", 100); """ qt_select_default """ SELECT * FROM ${tableName}; """ - assertTrue(triggerCompaction(backendId_to_backendIP[backend_id], backendId_to_backendHttpPort[backend_id], "cumulative").contains("Success")); - waitForCompaction(backendId_to_backendIP[backend_id], backendId_to_backendHttpPort[backend_id]) + trigger_and_wait_compaction(tableName, "cumulative") qt_select_default """ SELECT * FROM ${tableName}; """ - assertTrue(triggerCompaction(backendId_to_backendIP[backend_id], backendId_to_backendHttpPort[backend_id], "base").contains("Success")); - waitForCompaction(backendId_to_backendIP[backend_id], backendId_to_backendHttpPort[backend_id]) + trigger_and_wait_compaction(tableName, "base") qt_select_default """ SELECT * FROM ${tableName}; """ } finally { // try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/compaction/test_compaction_dup_keys.groovy b/regression-test/suites/compaction/test_compaction_dup_keys.groovy index 458185ba8069ac..bf435a734b51b8 100644 --- a/regression-test/suites/compaction/test_compaction_dup_keys.groovy +++ b/regression-test/suites/compaction/test_compaction_dup_keys.groovy @@ -28,7 +28,7 @@ suite("test_compaction_dup_keys") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -103,38 +103,7 @@ suite("test_compaction_dup_keys") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) @@ -145,7 +114,7 @@ suite("test_compaction_dup_keys") { (code, out, err) = curl("GET", tablet.CompactionStatus) logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) - + def tabletJson = parseJson(out.trim()) assert tabletJson.rowsets instanceof List for (String rowset in (List) tabletJson.rowsets) { diff --git a/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy b/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy index 2e34086172a553..02b5cfa95a61cb 100644 --- a/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy +++ b/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy @@ -28,7 +28,7 @@ suite("test_compaction_dup_keys_with_delete") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -104,7 +104,7 @@ suite("test_compaction_dup_keys_with_delete") { sql """ DELETE FROM ${tableName} where user_id <= 5 """ - + sql """ INSERT INTO ${tableName} VALUES (4, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) """ @@ -115,50 +115,18 @@ suite("test_compaction_dup_keys_with_delete") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) int rowCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId - + (code, out, err) = curl("GET", tablet.CompactionStatus) logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) - + def tabletJson = parseJson(out.trim()) assert tabletJson.rowsets instanceof List for (String rowset in (List) tabletJson.rowsets) { diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy index dd8d1dfb7d958b..e0a442da853850 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy @@ -28,7 +28,7 @@ suite("test_compaction_uniq_keys") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -137,7 +137,7 @@ suite("test_compaction_uniq_keys") { def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) - + int rowCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy index 6f65b896364044..8c978326d1e9f6 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy @@ -44,7 +44,7 @@ suite("test_compaction_uniq_keys_row_store", "p0") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -159,38 +159,7 @@ suite("test_compaction_uniq_keys_row_store", "p0") { checkValue() // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy index 18a46422d3292b..463597124c78c8 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy @@ -28,7 +28,7 @@ suite("test_compaction_uniq_keys_with_delete") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -119,37 +119,7 @@ suite("test_compaction_uniq_keys_with_delete") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) diff --git a/regression-test/suites/compaction/test_full_compaction.groovy b/regression-test/suites/compaction/test_full_compaction.groovy index 34742d41bf2460..3196bbae6e29d6 100644 --- a/regression-test/suites/compaction/test_full_compaction.groovy +++ b/regression-test/suites/compaction/test_full_compaction.groovy @@ -46,9 +46,9 @@ suite("test_full_compaction") { sql """ CREATE TABLE ${tableName} ( `user_id` INT NOT NULL, `value` INT NOT NULL) - UNIQUE KEY(`user_id`) - DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + UNIQUE KEY(`user_id`) + DISTRIBUTED BY HASH(`user_id`) + BUCKETS 1 PROPERTIES ("replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true", "enable_mow_light_delete" = "false", @@ -117,46 +117,9 @@ suite("test_full_compaction") { assert (rowsetCount == 7 * replicaNum) // trigger full compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - def times = 1 - - do{ - (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10) - - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for full compaction done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. - + rowsetCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId diff --git a/regression-test/suites/compaction/test_full_compaction_by_table_id.groovy b/regression-test/suites/compaction/test_full_compaction_by_table_id.groovy index 438a08be5158ee..3e016d84ff09b2 100644 --- a/regression-test/suites/compaction/test_full_compaction_by_table_id.groovy +++ b/regression-test/suites/compaction/test_full_compaction_by_table_id.groovy @@ -48,9 +48,9 @@ suite("test_full_compaction_by_table_id") { sql """ CREATE TABLE ${tableName} ( `user_id` INT NOT NULL, `value` INT NOT NULL) - UNIQUE KEY(`user_id`) - DISTRIBUTED BY HASH(`user_id`) - BUCKETS 8 + UNIQUE KEY(`user_id`) + DISTRIBUTED BY HASH(`user_id`) + BUCKETS 8 PROPERTIES ("replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true", "enable_mow_light_delete" = "false", @@ -116,52 +116,9 @@ suite("test_full_compaction_by_table_id") { } // trigger full compactions for all tablets by table id in ${tableName} - // TODO: get table id - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet"+tablet_info) - def table_id = tablet_info[0].TableId - backend_id = tablet.BackendId - def times = 1 - def code, out, err - do{ - (code, out, err) = be_run_full_compaction_by_table_id(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), table_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10) - - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for full compaction done - { - for (def tablet : tablets) { - boolean running = true - do { - Thread.sleep(1000) - def tablet_id = tablet.TabletId - backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. - for (def tablet : tablets) { int rowsetCount = 0 def (code, out, err) = curl("GET", tablet.CompactionStatus) diff --git a/regression-test/suites/compaction/test_single_replica_compaction.groovy b/regression-test/suites/compaction/test_single_replica_compaction.groovy index d8ff209b93cd43..05286a6fa197c6 100644 --- a/regression-test/suites/compaction/test_single_replica_compaction.groovy +++ b/regression-test/suites/compaction/test_single_replica_compaction.groovy @@ -22,7 +22,7 @@ suite("test_single_compaction_p2", "p2") { return; } def tableName = "test_single_replica_compaction" - + def calc_file_crc_on_tablet = { ip, port, tablet -> return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) } @@ -146,7 +146,7 @@ suite("test_single_compaction_p2", "p2") { // wait for update replica infos Thread.sleep(70000) - + // find the master be for single replica compaction Boolean found = false String master_backend_id; @@ -210,12 +210,12 @@ suite("test_single_compaction_p2", "p2") { // trigger master be to do compaction assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); + "full", tablet_id).contains("Success")); waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) // trigger follower be to fetch compaction result for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) } @@ -231,12 +231,12 @@ suite("test_single_compaction_p2", "p2") { // trigger master be to do compaction with delete assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); + "full", tablet_id).contains("Success")); waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) // trigger follower be to fetch compaction result for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); + assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) } diff --git a/regression-test/suites/compaction/test_time_series_compaction_policy.groovy b/regression-test/suites/compaction/test_time_series_compaction_policy.groovy index 2e8018f94a6a09..d211ba98bdd7e2 100644 --- a/regression-test/suites/compaction/test_time_series_compaction_policy.groovy +++ b/regression-test/suites/compaction/test_time_series_compaction_policy.groovy @@ -22,45 +22,6 @@ suite("test_time_series_compaction_polciy", "p0") { def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - def trigger_cumulative_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(1000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=3) - if (compactionStatus!="success") { - assertTrue(compactionStatus.contains("2000")) - continue; - } - assertEquals("success", compactionStatus) - } - } - - def wait_cumulative_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } def get_rowset_count = { tablets -> int rowsetCount = 0 @@ -109,7 +70,7 @@ suite("test_time_series_compaction_polciy", "p0") { sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """ sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """ sql """ INSERT INTO ${tableName} VALUES (100, "andy", "andy love apple", 100); """ - + qt_sql_1 """ select count() from ${tableName} """ //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus @@ -123,17 +84,14 @@ suite("test_time_series_compaction_polciy", "p0") { assert(false) } } - + // BUCKETS = 2 // before cumulative compaction, there are 17 * 2 = 34 rowsets. int rowsetCount = get_rowset_count.call(tablets); assert (rowsetCount == 34 * replicaNum) // trigger cumulative compactions for all tablets in table - trigger_cumulative_compaction_on_tablets.call(tablets) - - // wait for cumulative compaction done - wait_cumulative_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "cumulative") // after cumulative compaction, there is only 26 rowset. // 5 consecutive empty versions are merged into one empty version @@ -142,10 +100,7 @@ suite("test_time_series_compaction_polciy", "p0") { assert (rowsetCount == 26 * replicaNum) // trigger cumulative compactions for all tablets in ${tableName} - trigger_cumulative_compaction_on_tablets.call(tablets) - - // wait for cumulative compaction done - wait_cumulative_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "cumulative") // after cumulative compaction, there is only 22 rowset. // 26 - 4 = 22 @@ -159,10 +114,7 @@ suite("test_time_series_compaction_polciy", "p0") { sql """ alter table ${tableName} set ("time_series_compaction_file_count_threshold"="10")""" sql """sync""" // trigger cumulative compactions for all tablets in ${tableName} - trigger_cumulative_compaction_on_tablets.call(tablets) - - // wait for cumulative compaction done - wait_cumulative_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "cumulative") // after cumulative compaction, there is only 11 rowset. rowsetCount = get_rowset_count.call(tablets); diff --git a/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy b/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy index 8a661a00c13df1..89493e153436fe 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy @@ -116,37 +116,7 @@ suite("test_vertical_compaction_agg_keys") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) diff --git a/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy b/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy index 209f816785ca0a..60c954e94b2893 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy @@ -77,37 +77,7 @@ suite("test_vertical_compaction_agg_state") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) diff --git a/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy b/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy index ef6a0fe16ec151..7cdf6b67d406e3 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy @@ -28,7 +28,7 @@ suite("test_vertical_compaction_dup_keys") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -76,7 +76,7 @@ suite("test_vertical_compaction_dup_keys") { sql """ DELETE from ${tableName} where user_id <= 0 """ - + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ @@ -116,37 +116,7 @@ suite("test_vertical_compaction_dup_keys") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) diff --git a/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy b/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy index e89dcf8851f814..6bff003a028be2 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy @@ -113,37 +113,7 @@ suite("test_vertical_compaction_uniq_keys") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") def replicaNum = get_table_replica_num(tableName) logger.info("get table replica num: " + replicaNum) diff --git a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_skip_compaction.groovy b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_skip_compaction.groovy index b665ae4c19b539..031a2ec74f7231 100644 --- a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_skip_compaction.groovy +++ b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_skip_compaction.groovy @@ -54,7 +54,7 @@ suite("test_partial_update_skip_compaction", "nonConcurrent") { } logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); - def check_rs_metas = { expected_rs_meta_size, check_func -> + def check_rs_metas = { expected_rs_meta_size, check_func -> if (isCloudMode()) { return } diff --git a/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy b/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy index 4f720d51331bee..0a5484dd115687 100644 --- a/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy +++ b/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy @@ -19,7 +19,7 @@ suite("test_full_compaction_run_status","nonConcurrent") { def tableName = "full_compaction_run_status_test" - + // test successful group commit async load sql """ DROP TABLE IF EXISTS ${tableName} """ @@ -36,8 +36,8 @@ suite("test_full_compaction_run_status","nonConcurrent") { `k` int , `v` int , ) engine=olap - DISTRIBUTED BY HASH(`k`) - BUCKETS 2 + DISTRIBUTED BY HASH(`k`) + BUCKETS 2 properties( "replication_num" = "1", "disable_auto_compaction" = "true") diff --git a/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy b/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy index c6dfa6b885cf6c..c060888690c077 100644 --- a/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy +++ b/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy @@ -22,7 +22,7 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") { return } def tableName = "test_full_compaction_with_ordered_data" - + sql """ DROP TABLE IF EXISTS ${tableName} """ String backend_id; @@ -39,8 +39,8 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") { `v` int , ) engine=olap DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) - BUCKETS 3 + DISTRIBUTED BY HASH(k) + BUCKETS 3 properties( "replication_num" = "1", "disable_auto_compaction" = "true") @@ -70,38 +70,9 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") { assert (rowsetCount == 5 * replicaNum * 3) // trigger full compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - times = 1 - - do{ - (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10) - - } - - // wait for full compaction done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. - rowsetCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId @@ -155,38 +126,9 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") { assert (rowsetCount == 12 * replicaNum * 3) // trigger full compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - times = 1 - - do{ - (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10) - - } - - // wait for full compaction done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. - + rowsetCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId diff --git a/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy b/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy index 88c529d685dfe8..e1e1e051ec1beb 100644 --- a/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy +++ b/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy @@ -28,8 +28,8 @@ suite("test_variant_bloom_filter", "nonConcurrent") { table "${table_name}" // set http request header params - set 'read_json_by_line', 'true' - set 'format', 'json' + set 'read_json_by_line', 'true' + set 'format', 'json' set 'max_filter_ratio', '0.1' set 'memtable_on_sink_node', 'true' file file_name // import json file @@ -72,7 +72,7 @@ suite("test_variant_bloom_filter", "nonConcurrent") { getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); def tablets = sql_return_maparray """ show tablets from ${index_table}; """ - + for (def tablet in tablets) { int beforeSegmentCount = 0 String tablet_id = tablet.TabletId @@ -88,29 +88,7 @@ suite("test_variant_bloom_filter", "nonConcurrent") { } // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - assertEquals("success", compactJson.status.toLowerCase()) - } - - // wait for all compactions done - for (def tablet in tablets) { - Awaitility.await().atMost(3, TimeUnit.MINUTES).untilAsserted(() -> { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("compaction task for this tablet is not running", compactionStatus.msg.toLowerCase()) - return compactionStatus.run_status; - }); - } + trigger_and_wait_compaction(index_table, "full") for (def tablet in tablets) { int afterSegmentCount = 0 @@ -126,7 +104,7 @@ suite("test_variant_bloom_filter", "nonConcurrent") { } assertEquals(afterSegmentCount, 1) } - + try { GetDebugPoint().enableDebugPointForAllBEs("bloom_filter_must_filter_data") @@ -139,4 +117,4 @@ suite("test_variant_bloom_filter", "nonConcurrent") { } finally { GetDebugPoint().disableDebugPointForAllBEs("bloom_filter_must_filter_data") } -} \ No newline at end of file +} diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy index 2743071c4b4c11..2693047a34b90a 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy @@ -142,69 +142,13 @@ suite("test_index_change_with_cumulative_compaction") { sql """ CREATE INDEX idx_city ON ${tableName}(`city`) USING INVERTED """ wait_for_latest_op_on_table_finish(tableName, timeout) - // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative") - - String command = sb.toString() - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - // build index sql "build index idx_user_id on ${tableName}" sql "build index idx_date on ${tableName}" sql "build index idx_city on ${tableName}" - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + // trigger compactions for all tablets in ${tableName} + trigger_and_wait_compaction(tableName, "cumulative") int rowCount = 0 for (def tablet in tablets) { diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy index f8d807ffb563d9..ff485a71ec010e 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_full_compaction.groovy @@ -142,36 +142,6 @@ suite("test_index_change_with_full_compaction") { sql """ CREATE INDEX idx_city ON ${tableName}(`city`) USING INVERTED """ wait_for_latest_op_on_table_finish(tableName, timeout) - // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=full") - - String command = sb.toString() - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - // build index if (!isCloudMode()) { sql "build index idx_user_id on ${tableName}" @@ -179,34 +149,8 @@ suite("test_index_change_with_full_compaction") { sql "build index idx_city on ${tableName}" } - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + // trigger compactions for all tablets in ${tableName} + trigger_and_wait_compaction(tableName, "full") int rowCount = 0 for (def tablet in tablets) { diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy index 5d93c7d8570c21..03b98777a15514 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy @@ -24,7 +24,7 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); boolean disableAutoCompaction = false - + def set_be_config = { key, value -> for (String backend_id: backendId_to_backendIP.keySet()) { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) @@ -32,49 +32,6 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10) - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -110,7 +67,7 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { String backend_id; backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -179,10 +136,7 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { assert (rowsetCount == 7 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -210,10 +164,7 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { assert (rowsetCount == 7 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy index c7cb3d54fa1049..77ea822ba817ea 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy @@ -24,7 +24,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); boolean disableAutoCompaction = false - + def set_be_config = { key, value -> for (String backend_id: backendId_to_backendIP.keySet()) { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) @@ -32,49 +32,6 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10) - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -110,7 +67,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { String backend_id; backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -146,7 +103,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { UNIQUE KEY(`id`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( + PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", @@ -184,10 +141,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { assert (rowsetCount == 7 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -215,10 +169,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { assert (rowsetCount == 7 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy index 47ce08cb6f60da..eeafd6316790d9 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy @@ -24,7 +24,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); boolean disableAutoCompaction = false - + def set_be_config = { key, value -> for (String backend_id: backendId_to_backendIP.keySet()) { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) @@ -32,49 +32,6 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10) - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -111,7 +68,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { try { String backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -210,10 +167,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { assert (rowsetCount == 3 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets) @@ -243,10 +197,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { assert (rowsetCount == 2 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets) @@ -322,11 +273,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { assert (rowsetCount == 3 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) - + trigger_and_wait_compaction.call(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets) assert (rowsetCount == 1 * replicaNum) @@ -355,10 +302,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { assert (rowsetCount == 2 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets) diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy index cfe2f8004d1ec0..cfd222c0a03919 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy @@ -40,7 +40,7 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_segment_count = { tablet -> + def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId StringBuilder sb = new StringBuilder(); @@ -163,69 +163,11 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - String ip = backendId_to_backendIP.get(backend_id) - String port = backendId_to_backendHttpPort.get(backend_id) - int segment_count = calc_segment_count(tablet) - logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - check_nested_index_file(ip, port, tablet_id, 9, 3, "V2") - - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative") - - String command = sb.toString() - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - + trigger_and_wait_compaction(tableName, "cumulative") // wait for all compactions done for (def tablet in tablets) { - boolean running = true String tablet_id = tablet.TabletId backend_id = tablet.BackendId - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) int segment_count = calc_segment_count(tablet) diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy index 50eacbea7e39d5..0453d63a8106e4 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy @@ -40,7 +40,7 @@ suite("test_mor_table_with_format_v2", "inverted_index_format_v2") { assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_segment_count = { tablet -> + def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId StringBuilder sb = new StringBuilder(); @@ -165,69 +165,12 @@ suite("test_mor_table_with_format_v2", "inverted_index_format_v2") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - String ip = backendId_to_backendIP.get(backend_id) - String port = backendId_to_backendHttpPort.get(backend_id) - int segment_count = calc_segment_count(tablet) - logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - check_nested_index_file(ip, port, tablet_id, 10, 3, "V2") - - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=full") - - String command = sb.toString() - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } + trigger_and_wait_compaction(tableName, "full") // wait for all compactions done for (def tablet in tablets) { - boolean running = true String tablet_id = tablet.TabletId backend_id = tablet.BackendId - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) // after compaction, there are 1 rwoset in local mode and 2 rowsets in cloud mode. diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy index 717a094795b833..9c3a1d5aebaa61 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy @@ -40,7 +40,7 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_segment_count = { tablet -> + def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId StringBuilder sb = new StringBuilder(); @@ -163,69 +163,12 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - String ip = backendId_to_backendIP.get(backend_id) - String port = backendId_to_backendHttpPort.get(backend_id) - int segment_count = calc_segment_count(tablet) - logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - check_nested_index_file(ip, port, tablet_id, 9, 3, "V2") - - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative") - - String command = sb.toString() - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done + trigger_and_wait_compaction(tableName, "cumulative") + // check indexes for (def tablet in tablets) { boolean running = true String tablet_id = tablet.TabletId backend_id = tablet.BackendId - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy index 931074c352df79..e4e273e9fca171 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy @@ -46,7 +46,7 @@ suite("test_single_replica_compaction_with_format_v2", "inverted_index_format_v2 assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_segment_count = { tablet -> + def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId StringBuilder sb = new StringBuilder(); @@ -167,67 +167,10 @@ suite("test_single_replica_compaction_with_format_v2", "inverted_index_format_v2 def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} + trigger_and_wait_compaction(tableName, "cumulative") for (def tablet in tablets) { String tablet_id = tablet.TabletId backend_id = tablet.BackendId - String ip = backendId_to_backendIP.get(backend_id) - String port = backendId_to_backendHttpPort.get(backend_id) - check_nested_index_file(ip, port, tablet_id, 9, 3, "V2") - - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative") - - String command = sb.toString() - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://") - sb.append(backendId_to_backendIP.get(backend_id)) - sb.append(":") - sb.append(backendId_to_backendHttpPort.get(backend_id)) - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") diff --git a/regression-test/suites/inverted_index_p1/need_read_data/test_dup_table_inverted_index.groovy b/regression-test/suites/inverted_index_p1/need_read_data/test_dup_table_inverted_index.groovy index 44f96d3aa6d2fc..83b6069941941f 100644 --- a/regression-test/suites/inverted_index_p1/need_read_data/test_dup_table_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p1/need_read_data/test_dup_table_inverted_index.groovy @@ -16,7 +16,7 @@ // under the License. suite("test_dup_table_inverted_index", "p1") { - + // load data def load_data = { loadTableName, fileName -> streamLoad { @@ -59,51 +59,6 @@ suite("test_dup_table_inverted_index", "p1") { } } - def run_compaction = { compactionTableName -> - String backend_id; - - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - def tablets = sql_return_maparray """ show tablets from ${compactionTableName}; """ - - // run - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - times = 1 - - do{ - (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10) - - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - logger.info("Compaction was done automatically!") - } - } - - // wait - for (def tablet : tablets) { - boolean running = true - do { - Thread.sleep(1000) - def tablet_id = tablet.TabletId - backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - // def generate_dup_mow_sql = { tableName -> // List list = new ArrayList<>() // // FULLTEXT @@ -174,7 +129,7 @@ suite("test_dup_table_inverted_index", "p1") { // <= list.add("SELECT COUNT(*) FROM ${tableName} WHERE request <= 'POST'"); list.add("SELECT id FROM ${tableName} WHERE request <= 'POST' ORDER BY id LIMIT 2"); - + // > list.add("SELECT COUNT(*) FROM ${tableName} WHERE request > 'POST'"); list.add("SELECT id FROM ${tableName} WHERE request > 'POST' ORDER BY id LIMIT 2"); @@ -418,11 +373,11 @@ suite("test_dup_table_inverted_index", "p1") { list.add("SELECT COUNT(*) FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request <= 'POST';"); list.add("SELECT id FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request <= 'POST' ORDER BY id LIMIT 2;"); - list.add("SELECT COUNT(*) FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request > 'POST' ") - list.add("SELECT id FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request > 'POST' ORDER BY id LIMIT 2 ") + list.add("SELECT COUNT(*) FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request > 'POST' ") + list.add("SELECT id FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request > 'POST' ORDER BY id LIMIT 2 ") - list.add("SELECT COUNT(*) FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request >= 'POST' ") - list.add("SELECT id FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request >= 'POST' ORDER BY id LIMIT 2 ") + list.add("SELECT COUNT(*) FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request >= 'POST' ") + list.add("SELECT id FROM ${tableName} WHERE request MATCH_REGEXP 'GET' AND request >= 'POST' ORDER BY id LIMIT 2 ") // FULLTEXT MATCH_PHRASE_EDGE with others list.add("SELECT COUNT(*) FROM ${tableName} WHERE request MATCH_PHRASE_EDGE 'GET' AND request LIKE '%GET%' "); @@ -713,8 +668,8 @@ suite("test_dup_table_inverted_index", "p1") { list.add("SELECT `@timestamp` FROM ${tableName} WHERE id != 200 OR request LIKE '%GET%' OR NOT (size > 100 AND size LIKE '%0%') OR clientip > '17.0.0.0' ORDER BY id LIMIT 2;"); return list } - - + + try { sql """ set enable_match_without_inverted_index = true """ sql """ set enable_common_expr_pushdown = true """ @@ -723,7 +678,7 @@ suite("test_dup_table_inverted_index", "p1") { // create table sql """ CREATE TABLE IF NOT EXISTS dup_httplogs - ( + ( `id` bigint NOT NULL AUTO_INCREMENT(100), `@timestamp` int(11) NULL, `clientip` varchar(20) NULL, @@ -787,7 +742,7 @@ suite("test_dup_table_inverted_index", "p1") { logger.info("dup_result4 is {}", dup_result4); compare_result(dup_result3, dup_result4, all_dup_sql) - run_compaction.call(dupTableName) + trigger_and_wait_compaction(dupTableName, "full") def dup_result5 = execute_sql.call("enable_no_need_read_data_opt", "true", all_dup_sql) logger.info("dup_result5 is {}", dup_result5); def dup_result6 = execute_sql.call("enable_no_need_read_data_opt", "false", all_dup_sql) @@ -800,7 +755,7 @@ suite("test_dup_table_inverted_index", "p1") { // create table sql """ CREATE TABLE IF NOT EXISTS mow_httplogs - ( + ( `@timestamp` int(11) NULL, `clientip` varchar(20) NULL, `request` text NULL, @@ -847,7 +802,7 @@ suite("test_dup_table_inverted_index", "p1") { sql """ INSERT INTO ${mowTable} (`@timestamp`, clientip, request, status, size) VALUES (100, '1zyvBkWVAy5H0DDDaQnrp9MmAhfo0UNB9bOGyvSEX9MW66eymeDElVzVmsvUKHORwWg2hRLN7yd253zhXGs6k7PVPHy6uqtYTaxvHWm7njZYWtlqraDGE1fvrtnyUvlrGFPJZzkuj5FQfpYl6dV2bJHV0A3gpzogKSXJSyfH02ryb2ObKaJC6dnkMic00P6R3rUCBotrU7KAaGieALbFUXBGTvjsFKUvLgJexqAEJcKwiioTp0JH9Y3NUWgi2y5kclPmUG4xVrKHWXu7bI1MYJ1DCL1eCQCuqXmUf7eFyKcR6pzTFpkurcYq5R3SjprK13EkuLmVcDJMS8DNiLVCcCIOpHQMNgVFNLI7SPCl461FPOrL1xSuULAsLNjP5xgjjpn5Bu2dAug906fSVcnJwfHuuCly0sqYfNEI0Bd1IMiQOyoqA1pwdJMYMa6hig6imR3bJcnPptA6Fo1rooqzzt6gFnloqXeo9Hd9UB1F7QhfZO21QOZho19A5d12wcnOZCb3sRzomQqcPKSyvb17SxzoP9coAEpfXZEBrds60iuPjZaez79zeGP8X4KxuK1WwVDFw661zB6nvKCtNKFQqeKVMSFWAazw735TkQRGkjlif31f3uspvmBrLagvtjlfMoT138NnNxc2FbsK5wmssNfKFRk9zNg629b46rX7qLnC3ItPYgXyPSFqSF7snjqOUHJpzvcPhyY7tuDZVW2VTd3OtRdjdlAwHbSUrI5jWI1BCeP8cObIsOjd5', '1zyvBkWVAy5H0DDDaQnrp9MmAhfo0UNB9bOGyvSEX9MW66eymeDElVzVmsvUKHORwWg2hRLN7yd253zhXGs6k7PVPHy6uqtYTaxvHWm7njZYWtlqraDGE1fvrtnyUvlrGFPJZzkuj5FQfpYl6dV2bJHV0A3gpzogKSXJSyfH02ryb2ObKaJC6dnkMic00P6R3rUCBotrU7KAaGieALbFUXBGTvjsFKUvLgJexqAEJcKwiioTp0JH9Y3NUWgi2y5kclPmUG4xVrKHWXu7bI1MYJ1DCL1eCQCuqXmUf7eFyKcR6pzTFpkurcYq5R3SjprK13EkuLmVcDJMS8DNiLVCcCIOpHQMNgVFNLI7SPCl461FPOrL1xSuULAsLNjP5xgjjpn5Bu2dAug906fSVcnJwfHuuCly0sqYfNEI0Bd1IMiQOyoqA1pwdJMYMa6hig6imR3bJcnPptA6Fo1rooqzzt6gFnloqXeo9Hd9UB1F7QhfZO21QOZho19A5d12wcnOZCb3sRzomQqcPKSyvb17SxzoP9coAEpfXZEBrds60iuPjZaez79zeGP8X4KxuK1WwVDFw661zB6nvKCtNKFQqeKVMSFWAazw735TkQRGkjlif31f3uspvmBrLagvtjlfMoT138NnNxc2FbsK5wmssNfKFRk9zNg629b46rX7qLnC3ItPYgXyPSFqSF7snjqOUHJpzvcPhyY7tuDZVW2VTd3OtRdjdlAwHbSUrI5jWI1BCeP8cObIsOjd5', -2, -3) """ sql """ INSERT INTO ${mowTable} (`@timestamp`, clientip, request, status, size) VALUES (100, '1zyvBkWVAy5H0DDDaQnrp9MmAhfo0UNB9bOGyvSEX9MW66eymeDElVzVmsvUKHORwWg2hRLN7yd253zhXGs6k7PVPHy6uqtYTaxvHWm7njZYWtlqraDGE1fvrtnyUvlrGFPJZzkuj5FQfpYl6dV2bJHV0A3gpzogKSXJSyfH02ryb2ObKaJC6dnkMic00P6R3rUCBotrU7KAaGieALbFUXBGTvjsFKUvLgJexqAEJcKwiioTp0JH9Y3NUWgi2y5kclPmUG4xVrKHWXu7bI1MYJ1DCL1eCQCuqXmUf7eFyKcR6pzTFpkurcYq5R3SjprK13EkuLmVcDJMS8DNiLVCcCIOpHQMNgVFNLI7SPCl461FPOrL1xSuULAsLNjP5xgjjpn5Bu2dAug906fSVcnJwfHuuCly0sqYfNEI0Bd1IMiQOyoqA1pwdJMYMa6hig6imR3bJcnPptA6Fo1rooqzzt6gFnloqXeo9Hd9UB1F7QhfZO21QOZho19A5d12wcnOZCb3sRzomQqcPKSyvb17SxzoP9coAEpfXZEBrds60iuPjZaez79zeGP8X4KxuK1WwVDFw661zB6nvKCtNKFQqeKVMSFWAazw735TkQRGkjlif31f3uspvmBrLagvtjlfMoT138NnNxc2FbsK5wmssNfKFRk9zNg629b46rX7qLnC3ItPYgXyPSFqSF7snjqOUHJpzvcPhyY7tuDZVW2VTd3OtRdjdlAwHbSUrI5jWI1BCeP8cObIsOjd5', '1zyvBkWVAy5H0DDDaQnrp9MmAhfo0UNB9bOGyvSEX9MW66eymeDElVzVmsvUKHORwWg2hRLN7yd253zhXGs6k7PVPHy6uqtYTaxvHWm7njZYWtlqraDGE1fvrtnyUvlrGFPJZzkuj5FQfpYl6dV2bJHV0A3gpzogKSXJSyfH02ryb2ObKaJC6dnkMic00P6R3rUCBotrU7KAaGieALbFUXBGTvjsFKUvLgJexqAEJcKwiioTp0JH9Y3NUWgi2y5kclPmUG4xVrKHWXu7bI1MYJ1DCL1eCQCuqXmUf7eFyKcR6pzTFpkurcYq5R3SjprK13EkuLmVcDJMS8DNiLVCcCIOpHQMNgVFNLI7SPCl461FPOrL1xSuULAsLNjP5xgjjpn5Bu2dAug906fSVcnJwfHuuCly0sqYfNEI0Bd1IMiQOyoqA1pwdJMYMa6hig6imR3bJcnPptA6Fo1rooqzzt6gFnloqXeo9Hd9UB1F7QhfZO21QOZho19A5d12wcnOZCb3sRzomQqcPKSyvb17SxzoP9coAEpfXZEBrds60iuPjZaez79zeGP8X4KxuK1WwVDFw661zB6nvKCtNKFQqeKVMSFWAazw735TkQRGkjlif31f3uspvmBrLagvtjlfMoT138NnNxc2FbsK5wmssNfKFRk9zNg629b46rX7qLnC3ItPYgXyPSFqSF7snjqOUHJpzvcPhyY7tuDZVW2VTd3OtRdjdlAwHbSUrI5jWI1BCeP8cObIsOjd5', -2, -3) """ sql """ sync """ - + def all_mow_sql = generate_dup_mow_sql.call(mowTable) def mow_result1 = execute_sql.call("enable_no_need_read_data_opt", "true", all_mow_sql) logger.info("mow_result1 is {}", mow_result1); @@ -868,7 +823,7 @@ suite("test_dup_table_inverted_index", "p1") { logger.info("mow_result4 is {}", mow_result4); compare_result(mow_result3, mow_result4, all_mow_sql) - run_compaction.call(mowTable) + trigger_and_wait_compaction(mowTable, "full") def mow_result5 = execute_sql.call("enable_no_need_read_data_opt", "true", all_mow_sql) logger.info("mow_result5 is {}", mow_result5); def mow_result6 = execute_sql.call("enable_no_need_read_data_opt", "false", all_mow_sql) @@ -894,4 +849,4 @@ suite("test_dup_table_inverted_index", "p1") { } finally { sql """ set enable_match_without_inverted_index = true """ } -} \ No newline at end of file +} diff --git a/regression-test/suites/inverted_index_p1/test_show_data.groovy b/regression-test/suites/inverted_index_p1/test_show_data.groovy index a146d6e154f566..210c9477f57a7a 100644 --- a/regression-test/suites/inverted_index_p1/test_show_data.groovy +++ b/regression-test/suites/inverted_index_p1/test_show_data.groovy @@ -551,7 +551,7 @@ suite("test_show_data_with_compaction", "p1") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -664,42 +664,6 @@ suite("test_show_data_with_compaction", "p1") { return "wait_timeout" } - def run_compaction_and_wait = { tableName -> - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - - // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - logger.info("Compaction was done automatically!") - } else { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def create_table_run_compaction_and_wait = { test_name -> sql """ DROP TABLE IF EXISTS ${test_name}; """ sql """ @@ -726,14 +690,14 @@ suite("test_show_data_with_compaction", "p1") { sql """ INSERT INTO ${test_name} VALUES (3, "bason", "bason hate pear", 99); """ def data_size = wait_for_show_data_finish(test_name, 60000, 0) assertTrue(data_size != "wait_timeout") - run_compaction_and_wait(test_name) + trigger_and_wait_compaction(test_name, "full") data_size = wait_for_show_data_finish(test_name, 60000, data_size) assertTrue(data_size != "wait_timeout") return data_size } try { - + set_be_config.call("inverted_index_compaction_enable", "true") sql "DROP TABLE IF EXISTS ${tableWithIndexCompaction}" create_table_with_index.call(tableWithIndexCompaction) @@ -748,7 +712,7 @@ suite("test_show_data_with_compaction", "p1") { def with_index_size = wait_for_show_data_finish(tableWithIndexCompaction, 60000, 0) assertTrue(with_index_size != "wait_timeout") - run_compaction_and_wait(tableWithIndexCompaction) + trigger_and_wait_compaction(tableWithIndexCompaction, "full") with_index_size = wait_for_show_data_finish(tableWithIndexCompaction, 60000, with_index_size) assertTrue(with_index_size != "wait_timeout") @@ -764,7 +728,7 @@ suite("test_show_data_with_compaction", "p1") { def another_with_index_size = wait_for_show_data_finish(tableWithOutIndexCompaction, 60000, 0) assertTrue(another_with_index_size != "wait_timeout") - run_compaction_and_wait(tableWithOutIndexCompaction) + trigger_and_wait_compaction(tableWithOutIndexCompaction, "full") another_with_index_size = wait_for_show_data_finish(tableWithOutIndexCompaction, 60000, another_with_index_size) assertTrue(another_with_index_size != "wait_timeout") diff --git a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy index 317a0363230505..068b25b11b35f6 100644 --- a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy +++ b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy @@ -89,7 +89,7 @@ suite("test_map_load_and_compaction", "p0") { for (int i = 0; i < 5; ++i) { streamLoadJson.call(4063, dataFile1) } - + sql """sync""" // check result @@ -105,31 +105,7 @@ suite("test_map_load_and_compaction", "p0") { checkCompactionStatus.call(compactionStatus, 6) // trigger compaction - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - assertEquals("success", compactJson.status.toLowerCase()) - - def running = false - - // wait compactions done - do { - Thread.sleep(1000) - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def cs = parseJson(out.trim()) - assertEquals("success", cs.status.toLowerCase()) - running = cs.run_status - } while (running) - + trigger_and_wait_compaction(testTable, "cumulative") checkCompactionStatus.call(compactionStatus, 1) // finally check backend alive diff --git a/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy b/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy index 956df9371cbf10..71ae198fe00603 100644 --- a/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy +++ b/regression-test/suites/schema_change_p0/datev2/test_agg_keys_schema_change_datev2.groovy @@ -31,7 +31,7 @@ suite("test_agg_keys_schema_change_datev2") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -108,7 +108,7 @@ suite("test_agg_keys_schema_change_datev2") { } sql """sync""" qt_sql """select * from ${tbName} ORDER BY `datek1`;""" - do_compact(tbName) + trigger_and_wait_compaction(tbName, "cumulative") sql """sync""" qt_sql """select * from ${tbName} ORDER BY `datek1`;""" sql """delete from ${tbName} where `datev3` = '2022-01-01';""" @@ -159,7 +159,7 @@ suite("test_agg_keys_schema_change_datev2") { } sql """sync""" qt_sql """select * from ${tbName} ORDER BY `datek1`;""" - do_compact(tbName) + trigger_and_wait_compaction(tbName, "cumulative") sql """sync""" qt_sql """select * from ${tbName} ORDER BY `datek1`;""" sql """delete from ${tbName} where `datev3` = '2022-01-01 11:11:11';""" @@ -210,7 +210,7 @@ suite("test_agg_keys_schema_change_datev2") { } sql """sync""" qt_sql """select * from ${tbName} ORDER BY `datek1`;""" - do_compact(tbName) + trigger_and_wait_compaction(tbName, "cumulative") sql """sync""" qt_sql """select * from ${tbName} ORDER BY `datek1`;""" sql """delete from ${tbName} where `datev3` = '2022-01-01 11:11:11';""" diff --git a/regression-test/suites/schema_change_p0/datev2/test_schema_change_varchar_to_datev2.groovy b/regression-test/suites/schema_change_p0/datev2/test_schema_change_varchar_to_datev2.groovy index 89874a8679d4ad..920c6066baa41a 100644 --- a/regression-test/suites/schema_change_p0/datev2/test_schema_change_varchar_to_datev2.groovy +++ b/regression-test/suites/schema_change_p0/datev2/test_schema_change_varchar_to_datev2.groovy @@ -100,7 +100,7 @@ suite("test_schema_change_varchar_to_datev2") { sql """sync""" qt_sql_2 """select * from ${tbName} ORDER BY `k1`;""" - do_compact(tbName) + trigger_and_wait_compaction(tbName, "cumulative") sql """sync""" qt_sql_3 """select * from ${tbName} ORDER BY `k1`;""" sql """delete from ${tbName} where `k3` = '2020-01-02';""" diff --git a/regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy b/regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy index 88798db3c6c552..cf2af4bc760548 100644 --- a/regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy +++ b/regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy @@ -113,7 +113,7 @@ suite("test_agg_keys_schema_change_decimalv2", "nonConcurrent") { } sql """sync""" qt_sql2 """select * from ${tbName} ORDER BY 1,2,3,4;""" - do_compact(tbName) + trigger_and_wait_compaction(tbName, "cumulative") sql """sync""" qt_sql3 """select * from ${tbName} ORDER BY 1,2,3,4;""" diff --git a/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy b/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy index 41f2a32854a259..52b6f80b3aab01 100644 --- a/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy +++ b/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy @@ -97,7 +97,7 @@ suite("test_agg_keys_schema_change_decimalv3") { } sql """sync""" qt_sql """select * from ${tbName} ORDER BY `decimalv3k1`;""" - do_compact(tbName) + trigger_and_wait_compaction(tbName, "cumulative") sql """sync""" qt_sql """select * from ${tbName} ORDER BY `decimalv3k1`;""" sql """ alter table ${tbName} drop column `decimalv3v3` """ diff --git a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy index 68ee51c86ed319..f78fa6bf1bfcb4 100644 --- a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy @@ -22,7 +22,7 @@ suite ("test_agg_keys_schema_change") { def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ return jobStateResult[0][9] } - + def tableName = "schema_change_agg_keys_regression_test" try { @@ -72,7 +72,7 @@ suite ("test_agg_keys_schema_change") { // add key column case 1, not light schema change sql """ - ALTER table ${tableName} ADD COLUMN new_key_column INT default "2" + ALTER table ${tableName} ADD COLUMN new_key_column INT default "2" """ int max_try_time = 3000 @@ -171,31 +171,8 @@ suite ("test_agg_keys_schema_change") { """ // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - def backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - } + trigger_and_wait_compaction(tableName, "cumulative") - // wait for all compactions done - for (String[] tablet in tablets) { - boolean running = true - do { - Thread.sleep(100) - String tablet_id = tablet[0] - def backend_id = tablet[2] - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - qt_sc """ select count(*) from ${tableName} """ qt_sc """ SELECT * FROM schema_change_agg_keys_regression_test WHERE user_id=2 """ diff --git a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy index 4ed3e39cdbb8e3..b06a472290ea30 100644 --- a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy @@ -1,4 +1,4 @@ - + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -146,32 +146,8 @@ suite ("test_agg_mv_schema_change") { """ // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - def backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - //assertEquals(code, 0) - } + trigger_and_wait_compaction(tableName, "cumulative") - // wait for all compactions done - for (String[] tablet in tablets) { - boolean running = true - do { - Thread.sleep(100) - String tablet_id = tablet[0] - def backend_id = tablet[2] - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - qt_sc """ select count(*) from ${tableName} """ qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ @@ -182,4 +158,4 @@ suite ("test_agg_mv_schema_change") { } - + diff --git a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy index a24268eea264df..c178331f6bd1e6 100644 --- a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy @@ -1,4 +1,4 @@ - + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -154,32 +154,8 @@ suite ("test_agg_rollup_schema_change") { """ // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - def backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - //assertEquals(code, 0) - } + trigger_and_wait_compaction(tableName, "cumulative") - // wait for all compactions done - for (String[] tablet in tablets) { - boolean running = true - do { - Thread.sleep(100) - String tablet_id = tablet[0] - def backend_id = tablet[2] - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - qt_sc """ select count(*) from ${tableName} """ qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ diff --git a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy index 211bdaefe3c766..9983142986067a 100644 --- a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy @@ -78,7 +78,7 @@ suite ("test_agg_vals_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT MAX default "1" + ALTER table ${tableName} ADD COLUMN new_column INT MAX default "1" """ qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ @@ -136,31 +136,7 @@ suite ("test_agg_vals_schema_change") { """ // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - def backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - //assertEquals(code, 0) - } - - // wait for all compactions done - for (String[] tablet in tablets) { - boolean running = true - do { - Thread.sleep(100) - String tablet_id = tablet[0] - def backend_id = tablet[2] - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") qt_sc """ select count(*) from ${tableName} """ qt_sc """ SELECT * FROM ${tableName} WHERE user_id=2 """ diff --git a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy index e4eaedb500eec0..9ab83fb9f8de79 100644 --- a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy @@ -71,7 +71,7 @@ suite ("test_dup_keys_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ sql """ SELECT * FROM ${tableName} WHERE user_id=2 order by min_dwell_time """ diff --git a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy index 34c3fd505e05d1..c7cb9ee758c319 100644 --- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy @@ -94,7 +94,7 @@ suite ("test_dup_mv_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ waitForJob(tableName, 3000) diff --git a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy index fa7a30ba570337..3cd3115bd2f3cb 100644 --- a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy @@ -103,7 +103,7 @@ suite ("test_dup_rollup_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ sql """ SELECT * FROM ${tableName} WHERE user_id=2 order by min_dwell_time """ diff --git a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy index 7eb43c4c44a83e..9ef48c2d720e11 100644 --- a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy @@ -70,7 +70,7 @@ suite ("test_dup_vals_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ sql """ SELECT * FROM ${tableName} WHERE user_id=2 order by min_dwell_time """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy index 92f9f5267ba091..e1f65075394b2f 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy @@ -73,7 +73,7 @@ suite ("test_uniq_keys_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy index fd564d117d76db..b69f72cd2e615d 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy @@ -104,7 +104,7 @@ suite ("test_uniq_mv_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ @@ -165,15 +165,7 @@ suite ("test_uniq_mv_schema_change") { """ // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - //assertEquals(code, 0) - } + trigger_and_wait_compaction(tableName, "cumulative") // wait for all compactions done for (String[] tablet in tablets) { diff --git a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy index 2cd17c6f5e5bd8..f291fd98fac4a3 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy @@ -97,7 +97,7 @@ suite ("test_uniq_rollup_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ @@ -173,15 +173,7 @@ suite ("test_uniq_rollup_schema_change") { """ // compaction - String[][] tablets = sql """ show tablets from ${tableName}; """ - for (String[] tablet in tablets) { - String tablet_id = tablet[0] - backend_id = tablet[2] - logger.info("run compaction:" + tablet_id) - def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - //assertEquals(code, 0) - } + trigger_and_wait_compaction(tableName, "cumulative") // wait for all compactions done for (String[] tablet in tablets) { diff --git a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy index 631072a7f0be30..29c8a3ba57fd24 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy @@ -73,7 +73,7 @@ suite ("test_uniq_vals_schema_change") { // add column sql """ - ALTER table ${tableName} ADD COLUMN new_column INT default "1" + ALTER table ${tableName} ADD COLUMN new_column INT default "1" """ sql """ SELECT * FROM ${tableName} WHERE user_id=2 """ diff --git a/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy b/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy index 77f85cf2c8954f..74372b87b1ce77 100644 --- a/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_varchar_schema_change.groovy @@ -22,7 +22,7 @@ suite ("test_varchar_schema_change") { def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ return jobStateResult[0][9] } - + def tableName = "varchar_schema_change_regression_test" try { @@ -55,7 +55,7 @@ suite ("test_varchar_schema_change") { """ exception "Cannot shorten string length" } - + // test {//为什么第一次改没发生Nothing is changed错误？查看branch-1.2-lts代码 // sql """ alter table ${tableName} modify column c2 varchar(20) // """ @@ -146,8 +146,8 @@ suite ("test_varchar_schema_change") { qt_sc " select min(c2),max(c2) from ${tableName} group by c0 order by 1,2; " sleep(5000) - sql """ alter table ${tableName} - modify column c2 varchar(40), + sql """ alter table ${tableName} + modify column c2 varchar(40), modify column c3 varchar(6) DEFAULT '0' """ max_try_time = 1200 diff --git a/regression-test/suites/variant_github_events_p2/load.groovy b/regression-test/suites/variant_github_events_p2/load.groovy index 547b1cb7916a62..372e7250ff47f7 100644 --- a/regression-test/suites/variant_github_events_p2/load.groovy +++ b/regression-test/suites/variant_github_events_p2/load.groovy @@ -23,7 +23,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ def delta_time = 1000 def alter_res = "null" def useTime = 0 - + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> for(int t = delta_time; t <= OpTimeout; t += delta_time){ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" @@ -106,7 +106,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ logger.info("wait_for_last_build_index_on_table_running debug: " + alter_res) assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_running timeout") return "wait_timeout" - } + } def backendId_to_backendIP = [:] @@ -125,8 +125,8 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ table "${table_name}" // set http request header params - set 'read_json_by_line', 'true' - set 'format', 'json' + set 'read_json_by_line', 'true' + set 'format', 'json' set 'max_filter_ratio', '0.1' file file_name // import json file time 10000 // limit inflight 10s @@ -187,32 +187,9 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ def tablets = sql_return_maparray """ show tablets from github_events; """ // trigger compactions for all tablets in github_events - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction("github_events", "cumulative") - sql """set enable_match_without_inverted_index = false""" + sql """set enable_match_without_inverted_index = false""" sql """ set enable_common_expr_pushdown = true """ // filter by bloom filter qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1;""" @@ -220,4 +197,4 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ // query with inverted index qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where v["repo"]["name"] match 'xpressengine' order by 1;""" qt_sql """select count() from github_events where v["repo"]["name"] match 'apache' order by 1;""" -} \ No newline at end of file +} diff --git a/regression-test/suites/variant_p0/compaction/test_compaction.groovy b/regression-test/suites/variant_p0/compaction/test_compaction.groovy index 48d916e38e3909..d0d21ff35bfee6 100644 --- a/regression-test/suites/variant_p0/compaction/test_compaction.groovy +++ b/regression-test/suites/variant_p0/compaction/test_compaction.groovy @@ -26,7 +26,7 @@ suite("test_compaction_variant") { backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -48,7 +48,7 @@ suite("test_compaction_variant") { sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( k bigint, - v ${var_def} + v ${var_def} ) ${key_type} KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS ${buckets} @@ -136,7 +136,7 @@ suite("test_compaction_variant") { qt_sql_11 "SELECT * FROM ${tableName} ORDER BY k, cast(v as string); " qt_sql_22 "select k, cast(v['a'] as array) from ${tableName} where size(cast(v['a'] as array)) > 0 order by k" qt_sql_33 "select k, v['a'], cast(v['b'] as string) from ${tableName} where length(cast(v['b'] as string)) > 4 order by k" - qt_sql_55 "select cast(v['b'] as string), cast(v['b']['c'] as string) from ${tableName} where cast(v['b'] as string) != 'null' and cast(v['b'] as string) != '{}' order by k desc limit 10;" + qt_sql_55 "select cast(v['b'] as string), cast(v['b']['c'] as string) from ${tableName} where cast(v['b'] as string) != 'null' and cast(v['b'] as string) != '{}' order by k desc limit 10;" } } finally { diff --git a/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy b/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy index 69711018f91d93..84e17e2952d809 100644 --- a/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy +++ b/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy @@ -55,7 +55,7 @@ suite("test_compaction_sparse_column", "p1,nonConcurrent") { k bigint, v variant ) - DUPLICATE KEY(`k`) + DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", @@ -66,26 +66,26 @@ suite("test_compaction_sparse_column", "p1,nonConcurrent") { sql """insert into ${tableName} select 0, '{"a": 11245, "b" : 42000}' as json_str union all select 0, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "aaaaa"}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + sql """insert into ${tableName} select 1, '{"a": 11245, "b" : 42001}' as json_str union all select 1, '{"a": 1123}' as json_str union all select 1, '{"a" : 1234, "xxxx" : "bbbbb"}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + sql """insert into ${tableName} select 2, '{"a": 11245, "b" : 42002}' as json_str union all select 2, '{"a": 1123}' as json_str union all select 2, '{"a" : 1234, "xxxx" : "ccccc"}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + sql """insert into ${tableName} select 3, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}' as json_str union all select 3, '{"a": 1123}' as json_str union all select 3, '{"a": 11245, "b" : 42003}' as json_str from numbers("number" = "4096") limit 4096 ;""" sql """insert into ${tableName} select 4, '{"a" : 1234, "xxxx" : "eeeee", "point" : 5}' as json_str union all select 4, '{"a": 1123}' as json_str union all select 4, '{"a": 11245, "b" : 42004}' as json_str from numbers("number" = "4096") limit 4096 ;""" - - + + sql """insert into ${tableName} select 5, '{"a" : 1234, "xxxx" : "fffff", "point" : 42000}' as json_str union all select 5, '{"a": 1123}' as json_str union all select 5, '{"a": 11245, "b" : 42005}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + qt_select_b_bfcompact """ SELECT count(cast(v['b'] as int)) FROM ${tableName};""" qt_select_xxxx_bfcompact """ SELECT count(cast(v['xxxx'] as string)) FROM ${tableName};""" qt_select_point_bfcompact """ SELECT count(cast(v['point'] as bigint)) FROM ${tableName};""" diff --git a/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy b/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy index 83dc0a559e6fc5..015ab9baa1d03f 100644 --- a/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy +++ b/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy @@ -51,7 +51,7 @@ suite("test_compaction_extract_root", "p1,nonConcurrent") { k bigint, v variant ) - DUPLICATE KEY(`k`) + DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", @@ -64,22 +64,22 @@ suite("test_compaction_extract_root", "p1,nonConcurrent") { sql """insert into ${tableName} select 0, '{"a": 11245, "b" : {"state" : "open", "code" : 2}}' as json_str union all select 8, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "aaaaa"}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + sql """insert into ${tableName} select 1, '{"a": 11245, "b" : {"state" : "colse", "code" : 2}}' as json_str union all select 1, '{"a": 1123}' as json_str union all select 1, '{"a" : 1234, "xxxx" : "bbbbb"}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + sql """insert into ${tableName} select 2, '{"a": 11245, "b" : {"state" : "flat", "code" : 3}}' as json_str union all select 2, '{"a": 1123}' as json_str union all select 2, '{"a" : 1234, "xxxx" : "ccccc"}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + sql """insert into ${tableName} select 3, '{"a" : 1234, "xxxx" : 4, "point" : 5}' as json_str union all select 3, '{"a": 1123}' as json_str union all select 3, '{"a": 11245, "b" : 42003}' as json_str from numbers("number" = "4096") limit 4096 ;""" sql """insert into ${tableName} select 4, '{"a" : 1234, "xxxx" : "eeeee", "point" : 5}' as json_str union all select 4, '{"a": 1123}' as json_str union all select 4, '{"a": 11245, "b" : 42004}' as json_str from numbers("number" = "4096") limit 4096 ;""" - + sql """insert into ${tableName} select 5, '{"a" : 1234, "xxxx" : "fffff", "point" : 42000}' as json_str union all select 5, '{"a": 1123}' as json_str union all select 5, '{"a": 11245, "b" : 42005}' as json_str from numbers("number" = "4096") limit 4096 ;""" @@ -96,37 +96,7 @@ suite("test_compaction_extract_root", "p1,nonConcurrent") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - if (compactJson.status.toLowerCase() == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactJson.status.toLowerCase()) - } - } - - // wait for all compactions done - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } + trigger_and_wait_compaction(tableName, "cumulative") int rowCount = 0 for (def tablet in tablets) {