Merge branch 'branch-3.0' into branch-3.0_20241119_fix_flight_sql

be/src/cloud/config.cpp

@@ -75,5 +75,5 @@ DEFINE_mInt32(tablet_txn_info_min_expired_seconds, "120");
 
 DEFINE_mBool(enable_use_cloud_unique_id_from_fe, "true");
 
-DEFINE_mBool(enable_cloud_tablet_report, "true");
+DEFINE_mBool(enable_cloud_tablet_report, "false");
 } // namespace doris::config

be/src/olap/rowset/segment_v2/segment_writer.cpp

@@ -253,10 +253,10 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
         opts.data_page_size = storage_page_size;
     }
     DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size", {
-        auto table_id = DebugPoints::instance()->get_debug_param_or_default<int64_t>(
+        auto table_id = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
                 "VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
                 INT_MIN);
-        auto target_data_page_size = DebugPoints::instance()->get_debug_param_or_default<int64_t>(
+        auto target_data_page_size = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
                 "VerticalSegmentWriter._create_column_writer.storage_page_size",
                 "storage_page_size", INT_MIN);
         if (table_id == INT_MIN || target_data_page_size == INT_MIN) {

be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp

@@ -246,10 +246,10 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
         opts.data_page_size = storage_page_size;
     }
     DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size", {
-        auto table_id = DebugPoints::instance()->get_debug_param_or_default<int64_t>(
+        auto table_id = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
                 "VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
                 INT_MIN);
-        auto target_data_page_size = DebugPoints::instance()->get_debug_param_or_default<int64_t>(
+        auto target_data_page_size = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
                 "VerticalSegmentWriter._create_column_writer.storage_page_size",
                 "storage_page_size", INT_MIN);
         if (table_id == INT_MIN || target_data_page_size == INT_MIN) {

be/src/util/jvm_metrics.cpp

@@ -485,8 +485,8 @@ Status JvmStats::refresh(JvmMetrics* jvm_metrics) const {
     jvm_metrics->jvm_thread_count->set_value(threadCount < 0 ? 0 : threadCount);
 
     for (int i = 0; i < threadCount; i++) {
-        JNI_CALL_METHOD_CHECK_EXCEPTION(jobject, threadInfo, env,
-                                        GetObjectArrayElement((jobjectArray)threadInfos, i));
+        JNI_CALL_METHOD_CHECK_EXCEPTION_DELETE_REF(
+                jobject, threadInfo, env, GetObjectArrayElement((jobjectArray)threadInfos, i));
 
         if (threadInfo == nullptr) {
             continue;

docker/thirdparties/docker-compose/hive/scripts/README

@@ -1,9 +1,9 @@
 1. Download parquet file packages from:
-    https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/load/tpch1_parquet/tpch1.db.tar.gz
+    https://doris-regression-hk.oss-cn-hongkong.aliyuncs.com/regression/datalake/pipeline_data/tpch1.db.tar.gz
 
 2. Unzip and name it to "tpch1.db"
 
 3. Download paimon file from:
-    https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/paimon/paimon1.tar.gz
+    https://doris-regression-hk.oss-cn-hongkong.aliyuncs.com/regression/datalake/pipeline_data/paimon1.tar.gz
 
 4. Unzip and name it to "paimon1"

docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh

@@ -60,7 +60,7 @@ touch "$lockfile2"
 if [[ ! -d "/mnt/scripts/tpch1.db" ]]; then
     echo "/mnt/scripts/tpch1.db does not exist"
     cd /mnt/scripts/
-    curl -O https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/datalake/pipeline_data/tpch1.db.tar.gz
+    curl -O https://doris-regression-hk.oss-cn-hongkong.aliyuncs.com/regression/datalake/pipeline_data/tpch1.db.tar.gz
     tar -zxf tpch1.db.tar.gz
     rm -rf tpch1.db.tar.gz
     cd -

docker/thirdparties/run-thirdparties-docker.sh

@@ -446,7 +446,7 @@ start_hudi() {
         sudo rm -rf "${ROOT}"/docker-compose/hudi/hive-metastore-postgresql
         sudo mkdir "${ROOT}"/docker-compose/hudi/hive-metastore-postgresql
         if [[ ! -d "${ROOT}/docker-compose/hudi/scripts/hudi_docker_compose_attached_file" ]]; then
-            echo "Attached files does not exist, please download the https://doris-build-hk-1308700295.cos.ap-hongkong.myqcloud.com/regression/load/hudi/hudi_docker_compose_attached_file.zip file to the docker-compose/hudi/scripts/ directory and unzip it."
+            echo "Attached files does not exist, please download the https://doris-regression-hk.oss-cn-hongkong.aliyuncs.com/regression/load/hudi/hudi_docker_compose_attached_file.zip file to the docker-compose/hudi/scripts/ directory and unzip it."
             exit 1
         fi
         sudo docker compose -f "${ROOT}"/docker-compose/hudi/hudi.yaml --env-file "${ROOT}"/docker-compose/hudi/hadoop.env up -d

fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java

@@ -95,7 +95,7 @@ protected void setProperties(Map<String, String> properties) throws DdlException
 
         // the endpoint for ping need add uri scheme.
         String pingEndpoint = properties.get(S3Properties.ENDPOINT);
-        if (!pingEndpoint.startsWith("http://")) {
+        if (!pingEndpoint.startsWith("http://") && !pingEndpoint.startsWith("https://")) {
             pingEndpoint = "http://" + properties.get(S3Properties.ENDPOINT);
             properties.put(S3Properties.ENDPOINT, pingEndpoint);
             properties.put(S3Properties.Env.ENDPOINT, pingEndpoint);

fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java

@@ -1194,7 +1194,7 @@ public enum IgnoreSplitType {
     private boolean enableSyncRuntimeFilterSize = true;
 
     @VariableMgr.VarAttr(name = ENABLE_PARALLEL_RESULT_SINK, needForward = true, fuzzy = true)
-    private boolean enableParallelResultSink = true;
+    private boolean enableParallelResultSink = false;
 
     @VariableMgr.VarAttr(name = "sort_phase_num", fuzzy = true, needForward = true,
             description = {"如设置为1，则只生成1阶段sort，设置为2，则只生成2阶段sort，设置其它值，优化器根据代价选择sort类型",

fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java

@@ -222,4 +222,20 @@ public void testModifyProperties() throws Exception {
         modify.put("s3.access_key", "aaa");
         s3Resource.modifyProperties(modify);
     }
+
+    @Test
+    public void testHttpScheme() throws DdlException {
+        // if https:// is set, it should be replaced with http://
+        Map<String, String> properties = new HashMap<>();
+        properties.put("AWS_ENDPOINT", "https://aaa");
+        properties.put("AWS_REGION", "bbb");
+        properties.put("AWS_ROOT_PATH", "/path/to/root");
+        properties.put("AWS_ACCESS_KEY", "xxx");
+        properties.put("AWS_SECRET_KEY", "yyy");
+        properties.put("AWS_BUCKET", "test-bucket");
+        properties.put("s3_validity_check", "false");
+        S3Resource s3Resource = new S3Resource("s3_2");
+        s3Resource.setProperties(properties);
+        Assert.assertEquals(s3Resource.getProperty(S3Properties.ENDPOINT), "https://aaa");
+    }
 }

fe/fe-core/src/test/java/org/apache/doris/utframe/AnotherDemoTest.java

@@ -25,6 +25,7 @@
 import org.apache.doris.catalog.Table;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.FeConstants;
+import org.apache.doris.planner.ExchangeNode;
 import org.apache.doris.planner.OlapScanNode;
 import org.apache.doris.planner.PlanFragment;
 import org.apache.doris.planner.Planner;
@@ -120,7 +121,8 @@ public void testCreateDbAndTable() throws Exception {
         }
         // 5. query
         // TODO: we can not process real query for now. So it has to be a explain query
-        String queryStr = "explain select /*+ SET_VAR(disable_nereids_rules=PRUNE_EMPTY_PARTITION) */ * from db1.tbl1";
+        String queryStr = "explain select /*+ SET_VAR(disable_nereids_rules=PRUNE_EMPTY_PARTITION, "
+                + "enable_parallel_result_sink=true) */ * from db1.tbl1";
         StmtExecutor stmtExecutor = new StmtExecutor(ctx, queryStr);
         stmtExecutor.execute();
         Planner planner = stmtExecutor.planner();
@@ -129,5 +131,16 @@ public void testCreateDbAndTable() throws Exception {
         PlanFragment fragment = fragments.get(0);
         Assert.assertTrue(fragment.getPlanRoot() instanceof OlapScanNode);
         Assert.assertEquals(0, fragment.getChildren().size());
+
+        queryStr = "explain select /*+ SET_VAR(disable_nereids_rules=PRUNE_EMPTY_PARTITION, "
+                + "enable_parallel_result_sink=false) */ * from db1.tbl1";
+        stmtExecutor = new StmtExecutor(ctx, queryStr);
+        stmtExecutor.execute();
+        planner = stmtExecutor.planner();
+        fragments = planner.getFragments();
+        Assert.assertEquals(2, fragments.size());
+        fragment = fragments.get(0);
+        Assert.assertTrue(fragment.getPlanRoot() instanceof ExchangeNode);
+        Assert.assertEquals(1, fragment.getChildren().size());
     }
 }

fe/fe-core/src/test/java/org/apache/doris/utframe/DemoTest.java

@@ -26,6 +26,7 @@
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Table;
 import org.apache.doris.common.FeConstants;
+import org.apache.doris.planner.ExchangeNode;
 import org.apache.doris.planner.OlapScanNode;
 import org.apache.doris.planner.PlanFragment;
 import org.apache.doris.planner.Planner;
@@ -108,7 +109,8 @@ public void testCreateDbAndTable() throws Exception {
 
         // 7. query
         // TODO: we can not process real query for now. So it has to be a explain query
-        String queryStr = "explain select /*+ SET_VAR(disable_nereids_rules=PRUNE_EMPTY_PARTITION) */ * from db1.tbl1";
+        String queryStr = "explain select /*+ SET_VAR(disable_nereids_rules=PRUNE_EMPTY_PARTITION, "
+                + "enable_parallel_result_sink=true) */ * from db1.tbl1";
         StmtExecutor stmtExecutor = new StmtExecutor(connectContext, queryStr);
         stmtExecutor.execute();
         Planner planner = stmtExecutor.planner();
@@ -117,5 +119,16 @@ public void testCreateDbAndTable() throws Exception {
         PlanFragment fragment = fragments.get(0);
         Assertions.assertTrue(fragment.getPlanRoot() instanceof OlapScanNode);
         Assertions.assertEquals(0, fragment.getChildren().size());
+
+        queryStr = "explain select /*+ SET_VAR(disable_nereids_rules=PRUNE_EMPTY_PARTITION, "
+                + "enable_parallel_result_sink=false) */ * from db1.tbl1";
+        stmtExecutor = new StmtExecutor(connectContext, queryStr);
+        stmtExecutor.execute();
+        planner = stmtExecutor.planner();
+        fragments = planner.getFragments();
+        Assertions.assertEquals(2, fragments.size());
+        fragment = fragments.get(0);
+        Assertions.assertTrue(fragment.getPlanRoot() instanceof ExchangeNode);
+        Assertions.assertEquals(1, fragment.getChildren().size());
     }
 }

regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy

@@ -669,7 +669,7 @@ class Config {
             if (config.s3Source == "aliyun") {
                 s3BucketName = "doris-regression-hk"
             } else if (config.s3Source == "aliyun-internal") {
-                s3BucketName = "doris-regression"
+                s3BucketName = "doris-regression-bj"
             } else if (config.s3Source == "tencent") {
                 s3BucketName = "doris-build-1308700295"
             } else if (config.s3Source == "huawei") {

regression-test/pipeline/cloud_p0/run.sh

@@ -34,6 +34,8 @@ if [[ -z "${teamcity_build_checkoutDir}" ]]; then echo "ERROR: env teamcity_buil
 if [[ -z "${pr_num_from_trigger}" ]]; then echo "ERROR: env pr_num_from_trigger not set" && exit 1; fi
 if [[ -z "${commit_id_from_trigger}" ]]; then echo "ERROR: env commit_id_from_trigger not set" && exit 1; fi
 if [[ -z "${s3SourceAk}" || -z "${s3SourceSk}" ]]; then echo "ERROR: env s3SourceAk or s3SourceSk not set" && exit 1; fi
+if [[ -z "${hwYunAk}" || -z "${hwYunSk}" ]]; then echo "WARNING: env hwYunAk or hwYunSk not set"; fi
+if [[ -z "${txYunAk}" || -z "${txYunSk}" ]]; then echo "WARNING: env txYunAk or txYunSk not set"; fi
 
 # shellcheck source=/dev/null
 source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
@@ -55,6 +57,10 @@ run() {
         echo # add a new line to prevent two config items from being combined, which will cause the error "No signature of method"
         echo "ak='${s3SourceAk}'"
         echo "sk='${s3SourceSk}'"
+        echo "hwYunAk='${hwYunAk:-}'"
+        echo "hwYunSk='${hwYunSk:-}'"
+        echo "txYunAk='${txYunAk:-}'"
+        echo "txYunSk='${txYunSk:-}'"
     } >>"${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
     cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy \
         "${teamcity_build_checkoutDir}"/regression-test/conf/

regression-test/pipeline/common/github-utils.sh

@@ -245,6 +245,7 @@ file_changed_fe_ut() {
     if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
     for af in ${all_files}; do
         if [[ "${af}" == 'fe'* ]] ||
+            [[ "${af}" == 'gensrc'* ]] ||
             [[ "${af}" == 'fe_plugins'* ]] ||
             [[ "${af}" == 'bin/start_fe.sh' ]] ||
             [[ "${af}" == 'docs/zh-CN/docs/sql-manual/'* ]] ||
@@ -262,6 +263,8 @@ file_changed_be_ut() {
     if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
     for af in ${all_files}; do
         if [[ "${af}" == 'be'* ]] ||
+            [[ "${af}" == 'gensrc'* ]] ||
+            [[ "${af}" == 'common/cpp'* ]] ||
             [[ "${af}" == 'contrib'* ]] ||
             [[ "${af}" == 'thirdparty'* ]] ||
             [[ "${af}" == 'bin/start_be.sh' ]] ||
@@ -280,6 +283,8 @@ file_changed_cloud_ut() {
     if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
     for af in ${all_files}; do
         if [[ "${af}" == 'cloud/src/'* ]] ||
+            [[ "${af}" == 'gensrc'* ]] ||
+            [[ "${af}" == 'common/cpp'* ]] ||
             [[ "${af}" == 'cloud/test/'* ]]; then
             echo "cloud-ut related file changed, return need" && return 0
         fi

regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy

@@ -99,7 +99,6 @@ suite("test_s3_tvf_with_resource", "p0") {
 
     // test outfile to s3
     def outfile_url = outfile_to_S3()
-    // outfile_url like: s3://doris-build-hk-1308700295/est_s3_tvf/export_test/exp_f2cb650bbb94431a-ab0bc3e6f3e89f04_*
 
     // 1. normal
     try {