[pick](branch-3.0) pick apache#44489 apache#44894 (apache#45031)

fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java

@@ -186,11 +186,11 @@ public Schema fetchArrowFlightSchema(int timeoutMs) {
     public void close() throws Exception {
         ctx.setCommand(MysqlCommand.COM_SLEEP);
         ctx.clear();
-        // TODO support query profile
         for (StmtExecutor asynExecutor : returnResultFromRemoteExecutor) {
             asynExecutor.finalizeQuery();
         }
         returnResultFromRemoteExecutor.clear();
+        executor.finalizeQuery();
         ConnectContext.remove();
     }
 }

regression-test/framework/pom.xml

@@ -75,6 +75,7 @@ under the License.
         <antlr.version>4.9.3</antlr.version>
         <hadoop.version>2.8.0</hadoop.version>
         <aws-java-sdk-s3.version>1.11.95</aws-java-sdk-s3.version>
+        <!-- Arrow 18 only supports jdk17 -->
         <arrow.version>17.0.0</arrow.version>
     </properties>
     <build>

regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy

@@ -1027,19 +1027,23 @@ class Config {
         return buildUrlWithDb(jdbcUrl, dbName)
     }
 
-    Connection getConnectionByArrowFlightSql(String dbName) {
+    Connection getConnectionByArrowFlightSqlDbName(String dbName) {
         Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver")
         String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost")
         String arrowFlightSqlPort = otherConfigs.get("extArrowFlightSqlPort")
         String arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
                 "/?useServerPrepStmts=false&useSSL=false&useEncryption=false"
-        // TODO jdbc:arrow-flight-sql not support connect db
-        String dbUrl = buildUrlWithDbImpl(arrowFlightSqlUrl, dbName)
+        // Arrow 17.0.0-rc03 support jdbc:arrow-flight-sql connect db
+        // https://github.com/apache/arrow/issues/41947
+        if (dbName?.trim()) {
+            arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
+                "/catalog=" + dbName + "?useServerPrepStmts=false&useSSL=false&useEncryption=false"
+        }
         tryCreateDbIfNotExist(dbName)
-        log.info("connect to ${dbUrl}".toString())
+        log.info("connect to ${arrowFlightSqlUrl}".toString())
         String arrowFlightSqlJdbcUser = otherConfigs.get("extArrowFlightSqlUser")
         String arrowFlightSqlJdbcPassword = otherConfigs.get("extArrowFlightSqlPassword")
-        return DriverManager.getConnection(dbUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword)
+        return DriverManager.getConnection(arrowFlightSqlUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword)
     }
 
     Connection getDownstreamConnection() {

regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy

@@ -156,7 +156,7 @@ class SuiteContext implements Closeable {
         def threadConnInfo = threadArrowFlightSqlConn.get()
         if (threadConnInfo == null) {
             threadConnInfo = new ConnectionInfo()
-            threadConnInfo.conn = config.getConnectionByArrowFlightSql(dbName)
+            threadConnInfo.conn = config.getConnectionByArrowFlightSqlDbName(dbName)
             threadConnInfo.username = config.jdbcUser
             threadConnInfo.password = config.jdbcPassword
             threadArrowFlightSqlConn.set(threadConnInfo)

regression-test/suites/query_p0/aggregate/aggregate_count1.groovy

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-suite("aggregate_count1", "query,arrow_flight_sql") {
+suite("aggregate_count1", "query") {
     sql """ DROP TABLE IF EXISTS aggregate_count1 """
     sql """create table if not exists aggregate_count1 (
                 name varchar(128),

regression-test/suites/query_p0/aggregate/select_distinct.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("select_distinct", "arrow_flight_sql") {
+suite("select_distinct") {
     sql """DROP TABLE IF EXISTS decimal_a;"""
     sql """DROP TABLE IF EXISTS decimal_b;"""
     sql """DROP TABLE IF EXISTS decimal_c;"""

regression-test/suites/query_p0/casesensetive_column/join_with_column_casesensetive.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("join_with_column_casesensetive", "arrow_flight_sql") {
+suite("join_with_column_casesensetive") {
     def tables=["ad_order_data_v1","ad_order_data"]
 
     for (String table in tables) {

regression-test/suites/query_p0/cast/test_cast.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_cast', "arrow_flight_sql") {
+suite('test_cast') {
     def date = "date '2020-01-01'"
     def datev2 = "datev2 '2020-01-01'"
     def datetime = "timestamp '2020-01-01 12:34:45'"

regression-test/suites/query_p0/except/test_query_except.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_query_except", "arrow_flight_sql") {
+suite("test_query_except") {
     // test query except, depend on query_test_data_load.groovy
     sql "use test_query_db"
     qt_select_except1 """

regression-test/suites/query_p0/group_concat/test_group_concat.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_group_concat", "query,p0,arrow_flight_sql") {
+suite("test_group_concat", "query,p0") {
     qt_select """
                 SELECT group_concat(k6) FROM test_query_db.test where k6='false'
               """

regression-test/suites/query_p0/grouping_sets/test_grouping_sets1.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_grouping_sets1", "arrow_flight_sql") {
+suite("test_grouping_sets1") {
     qt_select """
         select 
             col1

regression-test/suites/query_p0/having/having.groovy

@@ -19,7 +19,7 @@
 // /testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate
 // and modified by Doris.
 
-suite("having", "query,p0,arrow_flight_sql") {
+suite("having", "query,p0") {
     sql """DROP TABLE IF EXISTS supplier"""
     sql """CREATE TABLE `supplier` (
             `s_suppkey` int(11) NOT NULL,

regression-test/suites/query_p0/intersect/test_intersect.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_intersect", "arrow_flight_sql") {
+suite("test_intersect") {
     qt_select """
                 SELECT * FROM (SELECT k1 FROM test_query_db.baseall
                     INTERSECT SELECT k1 FROM test_query_db.test) a ORDER BY k1

regression-test/suites/query_p0/join/test_join2.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_join2", "query,p0,arrow_flight_sql") {
+suite("test_join2", "query,p0") {
     def DBname = "regression_test_join2"
     def TBname1 = "J1_TBL"
     def TBname2 = "J2_TBL"

regression-test/suites/query_p0/join/test_left_join1.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_left_join1", "query,p0,arrow_flight_sql") {
+suite("test_left_join1", "query,p0") {
 
     def tableName = "test_left_join1"
     sql """drop table if exists ${tableName}"""

regression-test/suites/query_p0/join/test_nestedloop_outer_join.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_nestedloop_outer_join", "query_p0,arrow_flight_sql") {
+suite("test_nestedloop_outer_join", "query_p0") {
     def tbl1 = "test_nestedloop_outer_join1"
     def tbl2 = "test_nestedloop_outer_join2"
 

regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_partitioned_hash_join", "query,p0,arrow_flight_sql") {
+suite("test_partitioned_hash_join", "query,p0") {
     sql "drop table if exists test_partitioned_hash_join_l"
     sql "drop table if exists test_partitioned_hash_join_r"
     sql """ create table test_partitioned_hash_join_l (

regression-test/suites/query_p0/lateral_view/lateral_view.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("lateral_view", "arrow_flight_sql") {
+suite("lateral_view") {
     sql """ DROP TABLE IF EXISTS `test_explode_bitmap` """
 	sql """
 		CREATE TABLE `test_explode_bitmap` (

regression-test/suites/query_p0/limit/OffsetInSubqueryWithJoin.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_offset_in_subquery_with_join", "query,arrow_flight_sql") {
+suite("test_offset_in_subquery_with_join", "query") {
     // define a sql table
     def testTable = "test_offset_in_subquery_with_join"
 

regression-test/suites/query_p0/literal_view/lietral_test.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("literal_view_test", "arrow_flight_sql") {
+suite("literal_view_test") {
 
     sql """DROP TABLE IF EXISTS table1"""
 

regression-test/suites/query_p0/operator/test_set_operator.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_set_operators", "query,p0,arrow_flight_sql") {
+suite("test_set_operators", "query,p0") {
 
     sql """
         DROP TABLE IF EXISTS t1;

regression-test/suites/query_p0/operator/test_sort_operator.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_sort_operator", "query,p0,arrow_flight_sql") {
+suite("test_sort_operator", "query,p0") {
 
     sql """
         DROP TABLE IF EXISTS dim_org_ful;

regression-test/suites/query_p0/session_variable/test_default_limit.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_default_limit', "arrow_flight_sql") {
+suite('test_default_limit') {
     sql 'drop table if exists baseall'
     sql 'drop table if exists bigtable'
 

regression-test/suites/query_p0/show/test_show_create_table.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_show_create_table", "query,arrow_flight_sql") {
+suite("test_show_create_table", "query") {
     String tb_name = "tb_show_create_table";
     try {  
         sql """drop table if exists ${tb_name} """

regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_aggregate_all_functions", "arrow_flight_sql") {
+suite("test_aggregate_all_functions") {
 
     sql "set batch_size = 4096"
 

regression-test/suites/query_p0/sql_functions/case_function/test_case_function_null.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_case_function_null", "query,p0,arrow_flight_sql") {
+suite("test_case_function_null", "query,p0") {
     sql """ drop table if exists case_null0 """
     sql """ create table case_null0 (
                 `c0` decimalv3(17, 1) NULL,

regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy

@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_hash_function", "arrow_flight_sql") {
+suite("test_hash_function") {
     sql "set batch_size = 4096;"
     sql "set enable_profile = true;"
 

regression-test/suites/query_p0/sql_functions/ip_functions/test_ip_functions.groovy

@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_ip_functions", "arrow_flight_sql") {
+suite("test_ip_functions") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT ipv4_num_to_string(-1);"

regression-test/suites/query_p0/sql_functions/json_function/test_query_json_insert.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_query_json_insert", "query,arrow_flight_sql") {
+suite("test_query_json_insert", "query") {
     qt_sql "select json_insert('{\"a\": 1, \"b\": [2, 3]}', '\$', null);"
     qt_sql "select json_insert('{\"k\": [1, 2]}', '\$.k[0]', null, '\$.[1]', null);"
     def tableName = "test_query_json_insert"

regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy

@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_json_function", "arrow_flight_sql") {
+suite("test_json_function") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT get_json_double('{\"k1\":1.3, \"k2\":\"2\"}', \"\$.k1\");"

regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_conv", "arrow_flight_sql") {
+suite("test_conv") {
     qt_select "SELECT CONV(15,10,2)"
 
     sql """ drop table if exists test_conv; """

regression-test/suites/query_p0/sql_functions/search_functions/test_multi_string_search.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_multi_string_search", "arrow_flight_sql") {
+suite("test_multi_string_search") {
     def table_name = "test_multi_string_search_strings"
 
     sql """ DROP TABLE IF EXISTS ${table_name} """

regression-test/suites/query_p0/sql_functions/spatial_functions/test_gis_function.groovy

@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-suite("test_gis_function", "arrow_flight_sql") {
+suite("test_gis_function") {
     sql "set batch_size = 4096;"
 
     qt_sql "SELECT ST_AsText(ST_Point(24.7, 56.7));"

regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_string_function", "arrow_flight_sql") {
+suite("test_string_function") {
     sql "set batch_size = 4096;"
 
     qt_sql "select elt(0, \"hello\", \"doris\");"

regression-test/suites/query_p0/sql_functions/table_function/explode_split.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("explode_split", "arrow_flight_sql") {
+suite("explode_split") {
     def tableName = "test_lv_str"
 
     sql """ DROP TABLE IF EXISTS ${tableName} """

regression-test/suites/query_p0/sql_functions/test_alias_function.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite('test_alias_function', "arrow_flight_sql") {
+suite('test_alias_function') {
     sql '''
         CREATE ALIAS FUNCTION IF NOT EXISTS f1(DATETIMEV2(3), INT)
             with PARAMETER (datetime1, int1) as date_trunc(days_sub(datetime1, int1), 'day')'''

regression-test/suites/query_p0/sql_functions/test_predicate.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_predicate", "arrow_flight_sql") {
+suite("test_predicate") {
     sql """drop table if exists t1;"""
     sql """
             create table t1 (

regression-test/suites/query_p0/sql_functions/width_bucket_fuctions/test_width_bucket_function.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_width_bucket_function", "arrow_flight_sql") {
+suite("test_width_bucket_function") {
     qt_sql "select width_bucket(1, 2, 3, 2)"
     qt_sql "select width_bucket(null, 2, 3, 2)"
     qt_sql "select width_bucket(6, 2, 6, 4)"

regression-test/suites/query_p0/subquery/test_subquery2.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_subquery2", "arrow_flight_sql") {
+suite("test_subquery2") {
 
     sql """DROP TABLE IF EXISTS subquerytest2"""
 

regression-test/suites/query_p0/test_data_type_marks.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_data_type_marks", "arrow_flight_sql") {
+suite("test_data_type_marks") {
     def tbName = "org"
     sql "DROP TABLE IF EXISTS ${tbName}"
     sql """

regression-test/suites/query_p0/test_dict_with_null.groovy

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("dict_with_null", "query,arrow_flight_sql") {
+suite("dict_with_null", "query") {
     def tableName = "test_dict_with_null"
     sql "DROP TABLE IF EXISTS ${tableName}"
     sql """