From c46816a2bbaf51c2e90db9d908e5f4c8f9f385f8 Mon Sep 17 00:00:00 2001 From: wangbo Date: Wed, 26 Jun 2024 15:27:02 +0800 Subject: [PATCH 01/31] [test] fix workload policy test failed (#36837) --- .../test_workload_sched_policy.groovy | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy index ca0c00fc895bcc4..9539a5b244dce5a 100644 --- a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy +++ b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy @@ -148,36 +148,45 @@ suite("test_workload_sched_policy") { sql """drop user if exists test_workload_sched_user""" sql """create user test_workload_sched_user identified by '12345'""" sql """grant ADMIN_PRIV on *.*.* to test_workload_sched_user""" + sql "drop workload group if exists test_set_session_wg;" + sql "drop workload group if exists test_set_session_wg2;" + sql "create workload group test_set_session_wg properties('cpu_share'='1024');" + sql "create workload group test_set_session_wg2 properties('cpu_share'='1024');" + + sql "drop workload policy if exists test_set_var_policy;" + sql "drop workload policy if exists test_set_var_policy2;" // 1 create test_set_var_policy sql "create workload policy test_set_var_policy conditions(username='test_workload_sched_user')" + - "actions(set_session_variable 'parallel_pipeline_task_num=33');" + "actions(set_session_variable 'workload_group=test_set_session_wg');" def result1 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) { logger.info("begin sleep 15s to wait") Thread.sleep(15000) - sql "show variables like '%parallel_pipeline_task_num%';" + sql "show variables like 'workload_group';" } - assertEquals("parallel_pipeline_task_num", result1[0][0]) - assertEquals("33", result1[0][1]) + assertEquals("workload_group", result1[0][0]) + assertEquals("test_set_session_wg", result1[0][1]) // 2 create test_set_var_policy2 with higher priority sql "create workload policy test_set_var_policy2 conditions(username='test_workload_sched_user') " + - "actions(set_session_variable 'parallel_pipeline_task_num=22') properties('priority'='10');" + "actions(set_session_variable 'workload_group=test_set_session_wg2') properties('priority'='10');" def result2 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) { Thread.sleep(3000) - sql "show variables like '%parallel_pipeline_task_num%';" + sql "show variables like 'workload_group';" } - assertEquals("parallel_pipeline_task_num", result2[0][0]) - assertEquals("22", result2[0][1]) + assertEquals("workload_group", result2[0][0]) + assertEquals("test_set_session_wg2", result2[0][1]) // 3 disable test_set_var_policy2 sql "alter workload policy test_set_var_policy2 properties('enabled'='false');" def result3 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) { Thread.sleep(3000) - sql "show variables like '%parallel_pipeline_task_num%';" + sql "show variables like 'workload_group';" } - assertEquals("parallel_pipeline_task_num", result3[0][0]) - assertEquals("33", result3[0][1]) + assertEquals("workload_group", result3[0][0]) + assertEquals("test_set_session_wg", result3[0][1]) + sql "drop workload group if exists test_set_session_wg;" + sql "drop workload group if exists test_set_session_wg2;" sql "drop workload policy if exists test_set_var_policy;" sql "drop workload policy if exists test_set_var_policy2;" From f0b7422c2390a3367dffaf4cc93defb52a692377 Mon Sep 17 00:00:00 2001 From: amory Date: Wed, 26 Jun 2024 15:29:05 +0800 Subject: [PATCH 02/31] [fix](array)fix array with empty arg in be behavior (#36845) --- .../vec/functions/array/function_array_constructor.cpp | 10 +++++++--- .../data/nereids_function_p0/scalar_function/Array.out | 6 ++++++ .../nereids_function_p0/scalar_function/Array.groovy | 5 +++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/be/src/vec/functions/array/function_array_constructor.cpp b/be/src/vec/functions/array/function_array_constructor.cpp index af64027ec298f65..6a26725ac9d00d4 100644 --- a/be/src/vec/functions/array/function_array_constructor.cpp +++ b/be/src/vec/functions/array/function_array_constructor.cpp @@ -36,6 +36,7 @@ #include "vec/data_types/data_type.h" #include "vec/data_types/data_type_array.h" #include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" #include "vec/functions/function.h" #include "vec/functions/simple_function_factory.h" @@ -59,11 +60,14 @@ class FunctionArrayConstructor : public IFunction { bool use_default_implementation_for_nulls() const override { return false; } - size_t get_number_of_arguments() const override { return 1; } + size_t get_number_of_arguments() const override { return 0; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - DCHECK(!arguments.empty()) - << "function: " << get_name() << ", arguments should not be empty"; + // we accept with empty argument, like array(), which will be treated as array(UInt8) + if (arguments.empty()) { + return std::make_shared( + make_nullable(std::make_shared())); + } return std::make_shared(make_nullable(remove_nullable(arguments[0]))); } diff --git a/regression-test/data/nereids_function_p0/scalar_function/Array.out b/regression-test/data/nereids_function_p0/scalar_function/Array.out index bf55a289cedd83e..bc99cd9c3869332 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/Array.out +++ b/regression-test/data/nereids_function_p0/scalar_function/Array.out @@ -14445,3 +14445,9 @@ true ["2012-03-11 11:10:11", "2012-03-11 11:10:21"] ["2012-03-12 12:11:12", "2012-03-12 12:11:23"] +-- !array_empty_fe -- +[] + +-- !array_empty_be -- +[] + diff --git a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy index a786b9304388f48..5957ced51af9283 100644 --- a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy +++ b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy @@ -1276,4 +1276,9 @@ suite("nereids_scalar_fn_Array") { qt_sequence_datetime_hour """select sequence(kdtmv2s1, date_add(kdtmv2s1, interval kint-3 hour), interval kint hour) from fn_test order by kdtmv2s1;""" qt_sequence_datetime_minute """select sequence(kdtmv2s1, date_add(kdtmv2s1, interval kint+1 minute), interval kint minute) from fn_test order by kdtmv2s1;""" qt_sequence_datetime_second """select sequence(kdtmv2s1, date_add(kdtmv2s1, interval kint second), interval kint-1 second) from fn_test order by kdtmv2s1;""" + + // with array empty + qt_array_empty_fe """select array()""" + sql """ set debug_skip_fold_constant=true; """ + qt_array_empty_be """select array()""" } From 6d9d2dd242c39b134a41320a8645cf2a97693ac3 Mon Sep 17 00:00:00 2001 From: zfr95 <87513668+zfr9527@users.noreply.github.com> Date: Wed, 26 Jun 2024 15:55:41 +0800 Subject: [PATCH 03/31] [test](mtmv)Add group by aggregate negative case (#36562) --- .../dimension_join_agg_negative.groovy | 480 ++++++++++++++++++ 1 file changed, 480 insertions(+) create mode 100644 regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy new file mode 100644 index 000000000000000..14f527344b0dcc1 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy @@ -0,0 +1,480 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +This file is used specifically to test the negative presence of AGGs under joins. + */ +suite("dimension_join_agg_negative") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + + sql """ + drop table if exists orders_negative + """ + + sql """CREATE TABLE `orders_negative` ( + `o_orderkey` BIGINT NULL, + `o_custkey` INT NULL, + `o_orderstatus` VARCHAR(1) NULL, + `o_totalprice` DECIMAL(15, 2) NULL, + `o_orderpriority` VARCHAR(15) NULL, + `o_clerk` VARCHAR(15) NULL, + `o_shippriority` INT NULL, + `o_comment` VARCHAR(79) NULL, + `o_orderdate` DATE not NULL + ) ENGINE=OLAP + DUPLICATE KEY(`o_orderkey`, `o_custkey`) + COMMENT 'OLAP' + auto partition by range (date_trunc(`o_orderdate`, 'day')) () + DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + sql """ + drop table if exists lineitem_negative + """ + + sql """CREATE TABLE `lineitem_negative` ( + `l_orderkey` BIGINT NULL, + `l_linenumber` INT NULL, + `l_partkey` INT NULL, + `l_suppkey` INT NULL, + `l_quantity` DECIMAL(15, 2) NULL, + `l_extendedprice` DECIMAL(15, 2) NULL, + `l_discount` DECIMAL(15, 2) NULL, + `l_tax` DECIMAL(15, 2) NULL, + `l_returnflag` VARCHAR(1) NULL, + `l_linestatus` VARCHAR(1) NULL, + `l_commitdate` DATE NULL, + `l_receiptdate` DATE NULL, + `l_shipinstruct` VARCHAR(25) NULL, + `l_shipmode` VARCHAR(10) NULL, + `l_comment` VARCHAR(44) NULL, + `l_shipdate` DATE not NULL + ) ENGINE=OLAP + DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey ) + COMMENT 'OLAP' + auto partition by range (date_trunc(`l_shipdate`, 'day')) () + DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + sql""" + insert into orders_negative values + (null, 1, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (1, null, 'ok', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), + (1, 2, 'ok', null, 'a', 'b', 1, 'yy', '2023-10-20'), + (2, 3, 'ok', 109.2, null,'d',2, 'mm', '2023-10-21'), + (3, 1, 'ok', 99.5, 'a', null, 1, 'yy', '2023-10-22'), + (1, 3, 'ok', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), + (2, 1, 'ok', 109.2, 'c','d',2, null, '2023-10-18'), + (3, 2, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (4, 5, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); + """ + + sql """ + insert into lineitem_negative values + (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), + (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); + """ + + sql """analyze table orders_negative with sync;""" + sql """analyze table lineitem_negative with sync;""" + + def create_all_mv = { mv_name, mv_sql -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};""" + sql """DROP TABLE IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + ${mv_sql} + """ + } + + // left join + agg (function + group by + +-*/ + filter) + def left_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate, t1.o_shippriority, t1.o_orderkey""" + + def left_query_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate, t1.o_shippriority, t1.o_orderkey, t1.o_custkey""" + + def left_mv_stmt_2 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1 + """ + def left_query_stmt_2 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1, t1.o_custkey, t1.o_custkey + """ + + def left_mv_stmt_3 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative where o_orderkey > 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1 + """ + def left_query_stmt_3 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative where o_orderkey > 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1, t1.o_custkey + """ + + def left_mv_stmt_4 = """select t1.o_orderdate, (t1.o_orderkey+t1.col2) as col3, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1, count(*) as col2 from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,col3, t1.col1 + """ + def left_query_stmt_4 = """select t1.o_orderdate, (t1.o_orderkey+t1.col2) as col3, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1, count(*) as col2 from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,col3, t1.col1, t1.o_custkey + """ + + def left_mv_stmt_5 = """select t1.sum_total, max_total+min_total as col3, count_all + from (select o_orderkey, sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + sum(o_totalprice) + count(*) as col5, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by t1.sum_total, col3, count_all + """ + def left_query_stmt_5 = """select t1.sum_total, max_total+min_total as col3, count_all + from (select o_orderkey, sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + sum(o_totalprice) + count(*) as col5, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey) as t1 + left join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by t1.sum_total, col3, count_all, t1.col5 + """ + + def left_mv_stmt_6 = """select t1.l_shipdate, t1.l_quantity, t1.l_orderkey + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.l_quantity, t1.l_orderkey """ + def left_query_stmt_6 = """select t1.l_shipdate, t1.l_quantity, t1.l_orderkey + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.l_quantity, t1.l_orderkey, t1.l_partkey""" + + def left_mv_stmt_7 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey + """ + def left_query_stmt_7 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey, t1.l_partkey + """ + + def left_mv_stmt_8 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey + """ + def left_query_stmt_8 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey, t1.l_partkey + """ + + def left_mv_stmt_9 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + t1.col2 as col3 + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, col3 + """ + def left_query_stmt_9 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + t1.col2 as col3 + from orders_negative + left join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, col3, t1.l_partkey + """ + + def left_mv_stmt_10 = """select t1.sum_total, max_total+min_total as col3, count_all + from orders_negative + left join (select l_orderkey, sum(l_quantity) as sum_total, + max(l_quantity) as max_total, + min(l_quantity) as min_total, + count(*) as count_all, + sum(l_quantity) + count(*) as col5, + bitmap_union(to_bitmap(case when l_quantity > 1 and l_orderkey IN (1, 3) then l_partkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when l_quantity > 2 and l_orderkey IN (2) then l_partkey else null end)) as cnt_2 + from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.sum_total, col3, count_all + """ + def left_query_stmt_10 = """select t1.sum_total, max_total+min_total as col3, count_all + from orders_negative + left join (select l_orderkey, sum(l_quantity) as sum_total, + max(l_quantity) as max_total, + min(l_quantity) as min_total, + count(*) as count_all, + sum(l_quantity) + count(*) as col5, + bitmap_union(to_bitmap(case when l_quantity > 1 and l_orderkey IN (1, 3) then l_partkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when l_quantity > 2 and l_orderkey IN (2) then l_partkey else null end)) as cnt_2 + from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.sum_total, col3, count_all, t1.col5 + """ + + // inner join + agg (function + group by + +-*/ + filter) + def inner_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate, t1.o_shippriority, t1.o_orderkey""" + + def inner_query_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate, t1.o_shippriority, t1.o_orderkey, t1.o_custkey""" + + def inner_mv_stmt_2 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1 + """ + def inner_query_stmt_2 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1, t1.o_custkey, t1.o_custkey + """ + + def inner_mv_stmt_3 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative where o_orderkey > 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1 + """ + def inner_query_stmt_3 = """select t1.o_orderdate, t1.o_orderkey, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1 from orders_negative where o_orderkey > 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,t1.o_orderkey, t1.col1, t1.o_custkey + """ + + def inner_mv_stmt_4 = """select t1.o_orderdate, (t1.o_orderkey+t1.col2) as col3, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1, count(*) as col2 from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,col3, t1.col1 + """ + def inner_query_stmt_4 = """select t1.o_orderdate, (t1.o_orderkey+t1.col2) as col3, t1.col1 + from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, sum(o_shippriority) as col1, count(*) as col2 from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey, o_custkey, o_orderstatus, o_orderdate) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by + t1.o_orderdate,col3, t1.col1, t1.o_custkey + """ + + def inner_mv_stmt_5 = """select t1.sum_total, max_total+min_total as col3, count_all + from (select o_orderkey, sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + sum(o_totalprice) + count(*) as col5, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by t1.sum_total, col3, count_all + """ + def inner_query_stmt_5 = """select t1.sum_total, max_total+min_total as col3, count_all + from (select o_orderkey, sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + sum(o_totalprice) + count(*) as col5, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders_negative where o_orderkey >= 1 + 1 group by o_orderkey) as t1 + inner join lineitem_negative on lineitem_negative.l_orderkey = t1.o_orderkey + group by t1.sum_total, col3, count_all, t1.col5 + """ + + def inner_mv_stmt_6 = """select t1.l_shipdate, t1.l_quantity, t1.l_orderkey + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.l_quantity, t1.l_orderkey """ + def inner_query_stmt_6 = """select t1.l_shipdate, t1.l_quantity, t1.l_orderkey + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.l_quantity, t1.l_orderkey, t1.l_partkey""" + + def inner_mv_stmt_7 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey + """ + def inner_query_stmt_7 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey, t1.l_partkey + """ + + def inner_mv_stmt_8 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey + """ + def inner_query_stmt_8 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, t1.l_orderkey, t1.l_partkey + """ + + def inner_mv_stmt_9 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + t1.col2 as col3 + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, col3 + """ + def inner_query_stmt_9 = """select t1.l_shipdate, t1.col1, t1.l_orderkey + t1.col2 as col3 + from orders_negative + inner join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.l_shipdate, t1.col1, col3, t1.l_partkey + """ + + def inner_mv_stmt_10 = """select t1.sum_total, max_total+min_total as col3, count_all + from orders_negative + inner join (select l_orderkey, sum(l_quantity) as sum_total, + max(l_quantity) as max_total, + min(l_quantity) as min_total, + count(*) as count_all, + sum(l_quantity) + count(*) as col5, + bitmap_union(to_bitmap(case when l_quantity > 1 and l_orderkey IN (1, 3) then l_partkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when l_quantity > 2 and l_orderkey IN (2) then l_partkey else null end)) as cnt_2 + from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.sum_total, col3, count_all + """ + def inner_query_stmt_10 = """select t1.sum_total, max_total+min_total as col3, count_all + from orders_negative + inner join (select l_orderkey, sum(l_quantity) as sum_total, + max(l_quantity) as max_total, + min(l_quantity) as min_total, + count(*) as count_all, + sum(l_quantity) + count(*) as col5, + bitmap_union(to_bitmap(case when l_quantity > 1 and l_orderkey IN (1, 3) then l_partkey else null end)) as cnt_1, + bitmap_union(to_bitmap(case when l_quantity > 2 and l_orderkey IN (2) then l_partkey else null end)) as cnt_2 + from lineitem_negative where l_orderkey > 1 + 1 group by l_orderkey) as t1 + on t1.l_orderkey = orders_negative.o_orderkey + group by + t1.sum_total, col3, count_all, t1.col5 + """ + + def sql_list = [ + left_mv_stmt_1,left_mv_stmt_2,left_mv_stmt_3,left_mv_stmt_4,left_mv_stmt_5,left_mv_stmt_6,left_mv_stmt_7,left_mv_stmt_8,left_mv_stmt_9,left_mv_stmt_10, + inner_mv_stmt_1,inner_mv_stmt_2,inner_mv_stmt_3,inner_mv_stmt_4,inner_mv_stmt_5,inner_mv_stmt_6,inner_mv_stmt_7,inner_mv_stmt_8,inner_mv_stmt_9,inner_mv_stmt_10 + ] + def query_list = [ + left_query_stmt_1,left_query_stmt_2,left_query_stmt_3,left_query_stmt_4,left_query_stmt_5,left_query_stmt_6,left_query_stmt_7,left_query_stmt_8,left_query_stmt_9,left_query_stmt_10, + inner_query_stmt_1,inner_query_stmt_2,inner_query_stmt_3,inner_query_stmt_4,inner_query_stmt_5,inner_query_stmt_6,inner_query_stmt_7,inner_query_stmt_8,inner_query_stmt_9,inner_query_stmt_10 + ] + + for (int i = 0; i < sql_list.size(); i++) { + def origin_res = sql sql_list[i] + def query_res = sql query_list[i] + assert (origin_res.size() > 0) + assert (query_res.size() > 0) + } + + for (int i = 0; i < sql_list.size(); i++) { + logger.info("sql_list current index: " + (i + 1)) + + def mv_name = "mv_" + (i + 1) + + create_all_mv(mv_name, sql_list[i]) + def job_name = getJobName(db, mv_name) + waitingMTMVTaskFinished(job_name) + + explain { + sql("${query_list[i]}") + notContains "${mv_name}(${mv_name})" + } + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};""" + } + +} From 015f051f73a8fe6d0e50dc643b2fcd114838b465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Wed, 26 Jun 2024 16:51:39 +0800 Subject: [PATCH 04/31] [opt](Nereids) Optimize findValidItems method to handle circular dependencies (#36839) ## Proposed changes These optimizations allow the findValidItems method to correctly handle circular dependencies while maintaining the required output slots. The code is now more efficient and ensures that the necessary edges and items are preserved during the traversal process. --- .../doris/nereids/properties/FuncDeps.java | 25 +++++++++++++++---- .../rules/rewrite/EliminateGroupByKey.java | 2 +- .../nereids/properties/FuncDepsTest.java | 13 +++++----- .../rewrite/EliminateGroupByKeyTest.java | 21 +++++++++++----- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java index 6c1b302d7dc11d4..c17fd2eee57e1bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; /** * Function dependence items. @@ -96,11 +97,25 @@ private void dfs(Set parent, Set> visited, Set cir } } - // find item that not in a circle - private Set findValidItems() { + // Find items that are not part of a circular dependency. + // To keep the slots in requireOutputs, we need to always keep the edges that start with output slots. + // Note: We reduce the last edge in a circular dependency, + // so we need to traverse from parents that contain the required output slots. + private Set findValidItems(Set requireOutputs) { Set circleItem = new HashSet<>(); Set> visited = new HashSet<>(); - for (Set parent : edges.keySet()) { + Set> parentInOutput = edges.keySet().stream() + .filter(requireOutputs::containsAll) + .collect(Collectors.toSet()); + for (Set parent : parentInOutput) { + if (!visited.contains(parent)) { + dfs(parent, visited, circleItem); + } + } + Set> otherParent = edges.keySet().stream() + .filter(parent -> !parentInOutput.contains(parent)) + .collect(Collectors.toSet()); + for (Set parent : otherParent) { if (!visited.contains(parent)) { dfs(parent, visited, circleItem); } @@ -126,10 +141,10 @@ private Set findValidItems() { * @param slots the initial set of slot sets to be reduced * @return the minimal set of slot sets after applying all possible reductions */ - public Set> eliminateDeps(Set> slots) { + public Set> eliminateDeps(Set> slots, Set requireOutputs) { Set> minSlotSet = Sets.newHashSet(slots); Set> eliminatedSlots = new HashSet<>(); - Set validItems = findValidItems(); + Set validItems = findValidItems(requireOutputs); for (FuncDepsItem funcDepsItem : validItems) { if (minSlotSet.contains(funcDepsItem.dependencies) && minSlotSet.contains(funcDepsItem.determinants)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java index 9e205f858090bc1..fbe0988daff5bce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java @@ -91,7 +91,7 @@ LogicalAggregate eliminateGroupByKey(LogicalAggregate agg, return null; } - Set> minGroupBySlots = funcDeps.eliminateDeps(new HashSet<>(groupBySlots.values())); + Set> minGroupBySlots = funcDeps.eliminateDeps(new HashSet<>(groupBySlots.values()), requireOutput); Set removeExpression = new HashSet<>(); for (Entry> entry : groupBySlots.entrySet()) { if (!minGroupBySlots.contains(entry.getValue()) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsTest.java index 64df33acd602c8c..6b17305ed7a3e8c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.types.IntegerType; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -43,7 +44,7 @@ void testOneEliminate() { Set> slotSet = Sets.newHashSet(set1, set2, set3, set4); FuncDeps funcDeps = new FuncDeps(); funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s2)); - Set> slots = funcDeps.eliminateDeps(slotSet); + Set> slots = funcDeps.eliminateDeps(slotSet, ImmutableSet.of()); Set> expected = new HashSet<>(); expected.add(set1); expected.add(set3); @@ -58,7 +59,7 @@ void testChainEliminate() { funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s2)); funcDeps.addFuncItems(Sets.newHashSet(s2), Sets.newHashSet(s3)); funcDeps.addFuncItems(Sets.newHashSet(s3), Sets.newHashSet(s4)); - Set> slots = funcDeps.eliminateDeps(slotSet); + Set> slots = funcDeps.eliminateDeps(slotSet, ImmutableSet.of()); Set> expected = new HashSet<>(); expected.add(set1); Assertions.assertEquals(expected, slots); @@ -71,7 +72,7 @@ void testTreeEliminate() { funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s2)); funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s3)); funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s4)); - Set> slots = funcDeps.eliminateDeps(slotSet); + Set> slots = funcDeps.eliminateDeps(slotSet, ImmutableSet.of()); Set> expected = new HashSet<>(); expected.add(set1); Assertions.assertEquals(expected, slots); @@ -83,7 +84,7 @@ void testCircleEliminate1() { FuncDeps funcDeps = new FuncDeps(); funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s2)); funcDeps.addFuncItems(Sets.newHashSet(s2), Sets.newHashSet(s1)); - Set> slots = funcDeps.eliminateDeps(slotSet); + Set> slots = funcDeps.eliminateDeps(slotSet, ImmutableSet.of()); Set> expected = new HashSet<>(); expected.add(set1); expected.add(set3); @@ -99,7 +100,7 @@ void testCircleEliminate2() { funcDeps.addFuncItems(Sets.newHashSet(s2), Sets.newHashSet(s3)); funcDeps.addFuncItems(Sets.newHashSet(s3), Sets.newHashSet(s4)); funcDeps.addFuncItems(Sets.newHashSet(s4), Sets.newHashSet(s1)); - Set> slots = funcDeps.eliminateDeps(slotSet); + Set> slots = funcDeps.eliminateDeps(slotSet, ImmutableSet.of()); Set> expected = new HashSet<>(); expected.add(set1); Assertions.assertEquals(expected, slots); @@ -112,7 +113,7 @@ void testGraphEliminate1() { funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s2)); funcDeps.addFuncItems(Sets.newHashSet(s1), Sets.newHashSet(s3)); funcDeps.addFuncItems(Sets.newHashSet(s3), Sets.newHashSet(s4)); - Set> slots = funcDeps.eliminateDeps(slotSet); + Set> slots = funcDeps.eliminateDeps(slotSet, ImmutableSet.of()); Set> expected = new HashSet<>(); expected.add(set1); Assertions.assertEquals(expected, slots); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java index 203e902b3ebddf1..5a9e15cf4774d10 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKeyTest.java @@ -66,7 +66,7 @@ void testEliminateChain() { funcDeps.addFuncItems(set1, set2); funcDeps.addFuncItems(set2, set3); funcDeps.addFuncItems(set3, set4); - Set> slots = funcDeps.eliminateDeps(ImmutableSet.of(set1, set2, set3, set4)); + Set> slots = funcDeps.eliminateDeps(ImmutableSet.of(set1, set2, set3, set4), ImmutableSet.of()); Assertions.assertEquals(1, slots.size()); Assertions.assertEquals(set1, slots.iterator().next()); } @@ -78,7 +78,7 @@ void testEliminateCircle() { funcDeps.addFuncItems(set2, set3); funcDeps.addFuncItems(set3, set4); funcDeps.addFuncItems(set4, set1); - Set> slots = funcDeps.eliminateDeps(ImmutableSet.of(set1, set2, set3, set4)); + Set> slots = funcDeps.eliminateDeps(ImmutableSet.of(set1, set2, set3, set4), ImmutableSet.of()); Assertions.assertEquals(1, slots.size()); Assertions.assertEquals(set1, slots.iterator().next()); } @@ -89,7 +89,7 @@ void testEliminateTree() { funcDeps.addFuncItems(set1, set2); funcDeps.addFuncItems(set1, set3); funcDeps.addFuncItems(set1, set4); - Set> slots = funcDeps.eliminateDeps(ImmutableSet.of(set1, set2, set3, set4)); + Set> slots = funcDeps.eliminateDeps(ImmutableSet.of(set1, set2, set3, set4), ImmutableSet.of()); Assertions.assertEquals(1, slots.size()); Assertions.assertEquals(set1, slots.iterator().next()); } @@ -163,11 +163,20 @@ void testEliminateByPk() throws Exception { @Test void testEliminateByEqual() { PlanChecker.from(connectContext) - .analyze("select count(t1.name) from t1 as t1 join t1 as t2 on t1.name = t2.name group by t1.name, t2.name") + .analyze("select t1.name from t1 as t1 join t1 as t2 on t1.name = t2.name group by t1.name, t2.name") .rewrite() .printlnTree() .matches(logicalAggregate().when(agg -> - agg.getGroupByExpressions().size() == 1 && agg.getGroupByExpressions().get(0).toSql().equals("name"))); - } + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("name"))); + PlanChecker.from(connectContext) + .analyze("select t2.name from t1 as t1 join t1 as t2 " + + "on t1.name = t2.name group by t1.name, t2.name") + .rewrite() + .printlnTree() + .matches(logicalAggregate().when(agg -> + agg.getGroupByExpressions().size() == 1 + && agg.getGroupByExpressions().get(0).toSql().equals("name"))); + } } From 665bcc91a3660a06512c3b28f9c108ffadbe9aa5 Mon Sep 17 00:00:00 2001 From: AlexYue Date: Wed, 26 Jun 2024 17:07:02 +0800 Subject: [PATCH 05/31] [enhance](Azure) Check delete operation's response on Azure Blob Storage (#36800) We should check if the delete operation is successful or not. --- be/src/io/fs/azure_obj_storage_client.cpp | 110 ++++++++++++----- cloud/src/recycler/azure_obj_client.cpp | 138 ++++++++++++++++------ 2 files changed, 181 insertions(+), 67 deletions(-) diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 231113350f295aa..1984083f9fbe297 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -32,11 +33,15 @@ #include #include #include +#include +#include "common/exception.h" #include "common/logging.h" #include "common/status.h" #include "io/fs/obj_storage_client.h" +using namespace Azure::Storage::Blobs; + namespace { std::string wrap_object_storage_path_msg(const doris::io::ObjectStoragePathOptions& opts) { return fmt::format("bucket {}, key {}, prefix {}, path {}", opts.bucket, opts.key, opts.prefix, @@ -78,6 +83,46 @@ ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOption return {}; } +struct AzureBatchDeleter { + AzureBatchDeleter(BlobContainerClient* client, const ObjectStoragePathOptions& opts) + : _client(client), _batch(client->CreateBatch()), _opts(opts) {} + // Submit one blob to be deleted in `AzureBatchDeleter::execute` + void delete_blob(const std::string& blob_name) { + deferred_resps.emplace_back(_batch.DeleteBlob(blob_name)); + } + ObjectStorageResponse execute() { + if (deferred_resps.empty()) { + return {}; + } + auto resp = do_azure_client_call([&]() { _client->SubmitBatch(_batch); }, _opts); + if (resp.status.code != ErrorCode::OK) { + return resp; + } + + auto get_defer_response = [](const auto& defer) { + // DeferredResponse might throw exception + if (!defer.GetResponse().Value.Deleted) { + throw Exception(Status::IOError("Batch delete failed")); + } + }; + for (auto&& defer_response : deferred_resps) { + auto response = + do_azure_client_call([&]() { get_defer_response(defer_response); }, _opts); + if (response.status.code != ErrorCode::OK) { + return response; + } + } + + return {}; + } + +private: + BlobContainerClient* _client; + BlobContainerBatch _batch; + const ObjectStoragePathOptions& _opts; + std::vector> deferred_resps; +}; + // Azure would do nothing ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { @@ -136,7 +181,7 @@ ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { try { - Azure::Storage::Blobs::Models::BlobProperties properties = + Models::BlobProperties properties = _client->GetBlockBlobClient(opts.key).GetProperties().Value; return {.file_size = properties.BlobSize}; } catch (Azure::Core::RequestFailedException& e) { @@ -166,7 +211,7 @@ ObjectStorageResponse AzureObjStorageClient::get_object(const ObjectStoragePathO auto client = _client->GetBlockBlobClient(opts.key); return do_azure_client_call( [&]() { - Azure::Storage::Blobs::DownloadBlobToOptions download_opts; + DownloadBlobToOptions download_opts; Azure::Core::Http::HttpRange range {static_cast(offset), bytes_read}; download_opts.Range = range; auto resp = client.DownloadTo(reinterpret_cast(buffer), bytes_read, @@ -178,7 +223,7 @@ ObjectStorageResponse AzureObjStorageClient::get_object(const ObjectStoragePathO ObjectStorageResponse AzureObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, std::vector* files) { - auto get_file_file = [&](Azure::Storage::Blobs::ListBlobsPagedResponse& resp) { + auto get_file_file = [&](ListBlobsPagedResponse& resp) { std::ranges::transform(resp.Blobs, std::back_inserter(*files), [](auto&& blob_item) { return FileInfo { .file_name = blob_item.Name, .file_size = blob_item.BlobSize, .is_file = true}; @@ -186,7 +231,7 @@ ObjectStorageResponse AzureObjStorageClient::list_objects(const ObjectStoragePat }; return do_azure_client_call( [&]() { - Azure::Storage::Blobs::ListBlobsOptions list_opts; + ListBlobsOptions list_opts; list_opts.Prefix = opts.prefix; auto resp = _client->ListBlobs(list_opts); get_file_file(resp); @@ -210,16 +255,15 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects(const ObjectStorageP auto end = std::end(objs); while (begin != end) { - auto batch = _client->CreateBatch(); - auto chunkEnd = begin; - std::advance(chunkEnd, std::min(BlobBatchMaxOperations, - static_cast(std::distance(begin, end)))); - for (auto it = begin; it != chunkEnd; ++it) { - batch.DeleteBlob(*it); - } - begin = chunkEnd; - auto resp = do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts); - if (resp.status.code != ErrorCode::OK) { + auto deleter = AzureBatchDeleter(_client.get(), opts); + auto chunk_end = begin; + std::advance(chunk_end, std::min(BlobBatchMaxOperations, + static_cast(std::distance(begin, end)))); + + std::ranges::for_each(std::ranges::subrange(begin, chunk_end), + [&](const std::string& obj) { deleter.delete_blob(obj); }); + begin = chunk_end; + if (auto resp = deleter.execute(); resp.status.code != ErrorCode::OK) { return resp; } } @@ -227,32 +271,42 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects(const ObjectStorageP } ObjectStorageResponse AzureObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { - return do_azure_client_call([&]() { _client->DeleteBlob(opts.key); }, opts); + return do_azure_client_call( + [&]() { + auto resp = _client->DeleteBlob(opts.key); + if (!resp.Value.Deleted) { + throw Exception(Status::IOError("Delete azure blob failed")); + } + }, + opts); } ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( const ObjectStoragePathOptions& opts) { - Azure::Storage::Blobs::ListBlobsOptions list_opts; + ListBlobsOptions list_opts; list_opts.Prefix = opts.prefix; list_opts.PageSizeHint = BlobBatchMaxOperations; + auto delete_func = [&](const std::vector& blobs) -> ObjectStorageResponse { + auto deleter = AzureBatchDeleter(_client.get(), opts); + auto batch = _client->CreateBatch(); + for (auto&& blob_item : blobs) { + deleter.delete_blob(blob_item.Name); + } + if (auto response = deleter.execute(); response.status.code != ErrorCode::OK) { + return response; + } + return {}; + }; auto resp = _client->ListBlobs(list_opts); - auto batch = _client->CreateBatch(); - for (auto&& blob_item : resp.Blobs) { - batch.DeleteBlob(blob_item.Name); - } - auto response = do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts); - if (response.status.code != ErrorCode::OK) { + if (auto response = delete_func(resp.Blobs); response.status.code != ErrorCode::OK) { return response; } + while (!resp.NextPageToken->empty()) { - batch = _client->CreateBatch(); list_opts.ContinuationToken = resp.NextPageToken; resp = _client->ListBlobs(list_opts); - for (auto&& blob_item : resp.Blobs) { - batch.DeleteBlob(blob_item.Name); - } - auto response = do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts); - if (response.status.code != ErrorCode::OK) { + + if (auto response = delete_func(resp.Blobs); response.status.code != ErrorCode::OK) { return response; } } diff --git a/cloud/src/recycler/azure_obj_client.cpp b/cloud/src/recycler/azure_obj_client.cpp index 9ac7c414fee7d11..0d5e3f54827a587 100644 --- a/cloud/src/recycler/azure_obj_client.cpp +++ b/cloud/src/recycler/azure_obj_client.cpp @@ -29,10 +29,13 @@ #include #include #include +#include #include "common/logging.h" #include "recycler/obj_store_accessor.h" +using namespace Azure::Storage::Blobs; + namespace doris::cloud { constexpr size_t BlobBatchMaxOperations = 256; @@ -49,10 +52,57 @@ ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOption opts.prefix, opts.endpoint); LOG_WARNING(msg); return {-1, std::move(msg)}; + } catch (std::exception& e) { + auto msg = fmt::format( + "Azure request failed because {}, bucket {}, key {}, prefix {}, endpoint {}", + e.what(), opts.bucket, opts.key, opts.prefix, opts.endpoint); + LOG_WARNING(msg); + return {-1, std::move(msg)}; } return {}; } +struct AzureBatchDeleter { + AzureBatchDeleter(BlobContainerClient* client, const ObjectStoragePathOptions& opts) + : _client(client), _batch(client->CreateBatch()), _opts(opts) {} + // Submit one blob to be deleted in `AzureBatchDeleter::execute` + void delete_blob(const std::string& blob_name) { + deferred_resps.emplace_back(_batch.DeleteBlob(blob_name)); + } + ObjectStorageResponse execute() { + if (deferred_resps.empty()) { + return {}; + } + auto resp = do_azure_client_call([&]() { _client->SubmitBatch(_batch); }, _opts); + if (resp.ret != 0) { + return resp; + } + + auto get_defer_response = [](const auto& defer) { + // DeferredResponse might throw exception + if (!defer.GetResponse().Value.Deleted) { + throw std::runtime_error("Batch delete blobs failed"); + } + }; + + for (auto&& defer_response : deferred_resps) { + auto response = + do_azure_client_call([&]() { get_defer_response(defer_response); }, _opts); + if (response.ret != 0) { + return response; + } + } + + return {}; + } + +private: + BlobContainerClient* _client; + BlobContainerBatch _batch; + const ObjectStoragePathOptions& _opts; + std::vector> deferred_resps; +}; + ObjectStorageResponse AzureObjClient::put_object(const ObjectStoragePathOptions& opts, std::string_view stream) { auto client = _client->GetBlockBlobClient(opts.key); @@ -65,7 +115,7 @@ ObjectStorageResponse AzureObjClient::put_object(const ObjectStoragePathOptions& ObjectStorageResponse AzureObjClient::head_object(const ObjectStoragePathOptions& opts) { try { - Azure::Storage::Blobs::Models::BlobProperties properties = + Models::BlobProperties properties = _client->GetBlockBlobClient(opts.key).GetProperties().Value; return {}; } catch (Azure::Storage::StorageException& e) { @@ -95,7 +145,7 @@ ObjectStorageResponse AzureObjClient::list_objects(const ObjectStoragePathOption }; return do_azure_client_call( [&]() { - Azure::Storage::Blobs::ListBlobsOptions list_opts; + ListBlobsOptions list_opts; list_opts.Prefix = opts.prefix; auto resp = _client->ListBlobs(list_opts); get_object_meta(resp); @@ -123,16 +173,15 @@ ObjectStorageResponse AzureObjClient::delete_objects(const ObjectStoragePathOpti auto end = std::end(objs); while (begin != end) { - auto batch = _client->CreateBatch(); - auto chunkEnd = begin; - std::advance(chunkEnd, std::min(BlobBatchMaxOperations, - static_cast(std::distance(begin, end)))); - for (auto it = begin; it != chunkEnd; ++it) { - batch.DeleteBlob(*it); - } - begin = chunkEnd; - auto resp = do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts); - if (resp.ret != 0) { + auto deleter = AzureBatchDeleter(_client.get(), opts); + auto chunk_end = begin; + std::advance(chunk_end, std::min(BlobBatchMaxOperations, + static_cast(std::distance(begin, end)))); + + std::ranges::for_each(std::ranges::subrange(begin, chunk_end), + [&](const std::string& obj) { deleter.delete_blob(obj); }); + begin = chunk_end; + if (auto resp = deleter.execute(); resp.ret != 0) { return resp; } } @@ -140,31 +189,40 @@ ObjectStorageResponse AzureObjClient::delete_objects(const ObjectStoragePathOpti } ObjectStorageResponse AzureObjClient::delete_object(const ObjectStoragePathOptions& opts) { - return do_azure_client_call([&]() { _client->DeleteBlob(opts.key); }, opts); + return do_azure_client_call( + [&]() { + auto resp = _client->DeleteBlob(opts.key); + if (!resp.Value.Deleted) { + throw std::runtime_error("Delete azure blob failed"); + } + }, + opts); } ObjectStorageResponse AzureObjClient::delete_objects_recursively( const ObjectStoragePathOptions& opts) { - Azure::Storage::Blobs::ListBlobsOptions list_opts; + ListBlobsOptions list_opts; list_opts.Prefix = opts.prefix; list_opts.PageSizeHint = BlobBatchMaxOperations; + auto delete_func = [&](const std::vector& blobs) -> ObjectStorageResponse { + auto deleter = AzureBatchDeleter(_client.get(), opts); + auto batch = _client->CreateBatch(); + for (auto&& blob_item : blobs) { + deleter.delete_blob(blob_item.Name); + } + if (auto response = deleter.execute(); response.ret != 0) { + return response; + } + return {}; + }; auto resp = _client->ListBlobs(list_opts); - auto batch = _client->CreateBatch(); - for (auto&& blob_item : resp.Blobs) { - batch.DeleteBlob(blob_item.Name); - } - auto response = do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts); - if (response.ret != 0) { + if (auto response = delete_func(resp.Blobs); response.ret != 0) { return response; } while (!resp.NextPageToken->empty()) { - batch = _client->CreateBatch(); list_opts.ContinuationToken = resp.NextPageToken; resp = _client->ListBlobs(list_opts); - for (auto&& blob_item : resp.Blobs) { - batch.DeleteBlob(blob_item.Name); - } - auto response = do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts); + auto response = delete_func(resp.Blobs); if (response.ret != 0) { return response; } @@ -174,29 +232,31 @@ ObjectStorageResponse AzureObjClient::delete_objects_recursively( ObjectStorageResponse AzureObjClient::delete_expired(const ObjectStorageDeleteExpiredOptions& opts, int64_t expired_time) { - Azure::Storage::Blobs::ListBlobsOptions list_opts; + ListBlobsOptions list_opts; list_opts.Prefix = opts.path_opts.prefix; list_opts.PageSizeHint = BlobBatchMaxOperations; + auto delete_func = [&](const std::vector& blobs) -> ObjectStorageResponse { + auto deleter = AzureBatchDeleter(_client.get(), opts.path_opts); + auto batch = _client->CreateBatch(); + for (auto&& blob_item : blobs) { + if (blob_item.Details.LastModified.time_since_epoch().count() < expired_time) { + batch.DeleteBlob(blob_item.Name); + } + } + if (auto response = deleter.execute(); response.ret != 0) { + return response; + } + return {}; + }; auto resp = _client->ListBlobs(list_opts); - auto batch = _client->CreateBatch(); - for (auto&& blob_item : resp.Blobs) { - batch.DeleteBlob(blob_item.Name); - } - auto response = do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts.path_opts); + auto response = delete_func(resp.Blobs); if (response.ret != 0) { return response; } while (!resp.NextPageToken->empty()) { - batch = _client->CreateBatch(); list_opts.ContinuationToken = resp.NextPageToken; resp = _client->ListBlobs(list_opts); - for (auto&& blob_item : resp.Blobs) { - if (blob_item.Details.LastModified.time_since_epoch().count() < expired_time) { - batch.DeleteBlob(blob_item.Name); - } - } - auto response = - do_azure_client_call([&]() { _client->SubmitBatch(batch); }, opts.path_opts); + auto response = delete_func(resp.Blobs); if (response.ret != 0) { return response; } From 10ea08fa85624c2f6c4de62c33c2aa1e24c0c550 Mon Sep 17 00:00:00 2001 From: TengJianPing <18241664+jacktengg@users.noreply.github.com> Date: Wed, 26 Jun 2024 17:08:57 +0800 Subject: [PATCH 06/31] [fix](spill) fix memory orphan check failure of partitioned hash join (#36806) Also add fault injection regression test cases for spill. --- .../partitioned_hash_join_sink_operator.cpp | 22 +- .../partitioned_agg_fault_injection.groovy | 149 ++++++++++++ ...rtitioned_hash_join_fault_injection.groovy | 216 ++++++++++++++++++ .../spill/spill_sort_fault_injection.groovy | 158 +++++++++++++ 4 files changed, 535 insertions(+), 10 deletions(-) create mode 100644 regression-test/suites/fault_injection_p0/spill/partitioned_agg_fault_injection.groovy create mode 100644 regression-test/suites/fault_injection_p0/spill/partitioned_hash_join_fault_injection.groovy create mode 100644 regression-test/suites/fault_injection_p0/spill/spill_sort_fault_injection.groovy diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp index 81bc2253657086a..cb104cfc7cda0ec 100644 --- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp +++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp @@ -126,13 +126,8 @@ Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block(RuntimeSta _shared_state->shared_from_this(); auto query_id = state->query_id(); auto mem_tracker = state->get_query_ctx()->query_mem_tracker; - auto spill_func = [build_blocks = std::move(build_blocks), state, num_slots, this]() mutable { - Defer defer {[&]() { - // need to reset build_block here, or else build_block will be destructed - // after SCOPED_ATTACH_TASK_WITH_ID and will trigger memory_orphan_check failure - build_blocks.clear(); - }}; - + auto spill_func = [state, num_slots, + this](std::vector& build_blocks) mutable { auto& p = _parent->cast(); auto& partitioned_blocks = _shared_state->partitioned_build_blocks; std::vector> partitions_indexes(p._partition_count); @@ -216,9 +211,16 @@ Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block(RuntimeSta _dependency->set_ready(); }; - auto exception_catch_func = [spill_func, shared_state_holder, execution_context, state, - query_id, mem_tracker, this]() mutable { + auto exception_catch_func = [build_blocks = std::move(build_blocks), spill_func, + shared_state_holder, execution_context, state, query_id, + mem_tracker, this]() mutable { SCOPED_ATTACH_TASK_WITH_ID(mem_tracker, query_id); + Defer defer {[&]() { + // need to reset build_block here, or else build_block will be destructed + // after SCOPED_ATTACH_TASK_WITH_ID and will trigger memory_orphan_check failure + build_blocks.clear(); + }}; + std::shared_ptr execution_context_lock; auto shared_state_sptr = shared_state_holder.lock(); if (shared_state_sptr) { @@ -230,7 +232,7 @@ Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block(RuntimeSta } auto status = [&]() { - RETURN_IF_CATCH_EXCEPTION(spill_func()); + RETURN_IF_CATCH_EXCEPTION(spill_func(build_blocks)); return Status::OK(); }(); diff --git a/regression-test/suites/fault_injection_p0/spill/partitioned_agg_fault_injection.groovy b/regression-test/suites/fault_injection_p0/spill/partitioned_agg_fault_injection.groovy new file mode 100644 index 000000000000000..0cefbba06579023 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/spill/partitioned_agg_fault_injection.groovy @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("partitioned_agg_fault_injection", "nonConcurrent") { + multi_sql """ + use regression_test_tpch_unique_sql_zstd_p0; + set enable_force_spill=true; + set min_revocable_mem=1024; + """ + def test_sql = """ + select + l_orderkey, + l_suppkey, + l_partkey, + l_quantity, + l_extendedprice, + l_discount, + l_returnflag, + l_linestatus, + l_commitdate, + l_receiptdate, + l_shipinstruct, + l_shipmode, + sum(l_tax) +from + lineitem +group by + l_orderkey, + l_suppkey, + l_partkey, + l_quantity, + l_extendedprice, + l_discount, + l_returnflag, + l_linestatus, + l_commitdate, + l_receiptdate, + l_shipinstruct, + l_shipmode + """ + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::spill_block") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream spill_block failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::spill_block") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::prepare_spill") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream prepare_spill failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::prepare_spill") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::spill_eof") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream spill_eof failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::spill_eof") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::read_next_block") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream read_next_block failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::read_next_block") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_agg_sink::sink") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_agg_sink sink failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_agg_sink::sink") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_agg_sink::revoke_memory_submit_func") + sql test_sql + } catch(Exception e) { + assertTrue(e.getMessage().contains("fault_inject partitioned_agg_sink revoke_memory submit_func failed")); + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_agg_sink::revoke_memory_submit_func") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_agg_sink::revoke_memory_cancel") + sql test_sql + } catch(Exception e) { + assertTrue(e.getMessage().contains("fault_inject partitioned_agg_sink revoke_memory canceled")); + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_agg_sink::revoke_memory_cancel") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_agg_source::recover_spill_data") + sql test_sql + } catch(Exception e) { + assertTrue(e.getMessage().contains("fault_inject partitioned_agg_source recover_spill_data failed")); + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_agg_source::recover_spill_data") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_agg_source::merge_spill_data_cancel") + sql test_sql + } catch(Exception e) { + assertTrue(e.getMessage().contains("fault_inject partitioned_agg_source merge spill data canceled")); + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_agg_source::merge_spill_data_cancel") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_agg_source::submit_func") + sql test_sql + } catch(Exception e) { + assertTrue(e.getMessage().contains("fault_inject partitioned_agg_source submit_func failed")); + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_agg_source::submit_func") + } +} \ No newline at end of file diff --git a/regression-test/suites/fault_injection_p0/spill/partitioned_hash_join_fault_injection.groovy b/regression-test/suites/fault_injection_p0/spill/partitioned_hash_join_fault_injection.groovy new file mode 100644 index 000000000000000..72e5179c46ed236 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/spill/partitioned_hash_join_fault_injection.groovy @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("partitioned_hash_join_fault_injection", "nonConcurrent") { + multi_sql """ + use regression_test_tpch_unique_sql_zstd_p0; + set enable_force_spill=true; + set min_revocable_mem=1024; + """ + def test_sql = """ + SELECT + L_ORDERKEY, + L_COMMENT, + O_ORDERKEY, + L_QUANTITY, + L_SHIPINSTRUCT + FROM + lineitem, + orders + WHERE + L_ORDERKEY = o_orderkey + """ + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::spill_block") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream spill_block failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::spill_block") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::prepare_spill") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream prepare_spill failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::prepare_spill") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::spill_eof") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream spill_eof failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::spill_eof") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::read_next_block") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream read_next_block failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::read_next_block") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::revoke_unpartitioned_block_submit_func") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_sink revoke_unpartitioned_block submit_func failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::revoke_unpartitioned_block_submit_func") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::revoke_memory_submit_func") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_sink revoke_memory submit_func failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::revoke_memory_submit_func") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::revoke_memory_cancel") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_sink revoke_memory canceled")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::revoke_memory_cancel") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::sink") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_sink sink failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_sink::sink") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::spill_probe_blocks") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe spill_probe_blocks failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::spill_probe_blocks") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::spill_probe_blocks_cancel") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe spill_probe_blocks canceled")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::spill_probe_blocks_cancel") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::spill_probe_blocks_submit_func") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe spill_probe_blocks submit_func failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::spill_probe_blocks_submit_func") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_build_blocks") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe recover_build_blocks failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_build_blocks") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_build_blocks_cancel") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe recover_build_blocks canceled")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_build_blocks_cancel") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recovery_build_blocks_submit_func") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe recovery_build_blocks submit_func failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recovery_build_blocks_submit_func") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_probe_blocks") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe recover_probe_blocks failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_probe_blocks") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_probe_blocks_cancel") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe recover_probe_blocks canceled")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recover_probe_blocks_cancel") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recovery_probe_blocks_submit_func") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe recovery_probe_blocks submit_func failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::recovery_probe_blocks_submit_func") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::sink") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject partitioned_hash_join_probe sink failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::partitioned_hash_join_probe::sink") + } +} \ No newline at end of file diff --git a/regression-test/suites/fault_injection_p0/spill/spill_sort_fault_injection.groovy b/regression-test/suites/fault_injection_p0/spill/spill_sort_fault_injection.groovy new file mode 100644 index 000000000000000..ce9d9ee9dd7610e --- /dev/null +++ b/regression-test/suites/fault_injection_p0/spill/spill_sort_fault_injection.groovy @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("spill_sort_fault_injection", "nonConcurrent") { + multi_sql """ + use regression_test_tpch_unique_sql_zstd_p0; + set enable_force_spill=true; + set min_revocable_mem=1024; + """ + def test_sql = """ + select + l_orderkey, + l_linenumber, + l_partkey, + l_suppkey, + l_quantity, + l_extendedprice, + l_discount, + l_tax, + l_returnflag, + l_linestatus, + l_commitdate, + l_receiptdate, + l_shipinstruct, + l_shipmode, + l_shipdate +from + lineitem +order by + l_orderkey, + l_linenumber, + l_partkey, + l_suppkey, + l_quantity, + l_extendedprice, + l_discount, + l_tax, + l_returnflag, + l_linestatus, + l_commitdate, + l_receiptdate, + l_shipinstruct, + l_shipmode; + """ + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::spill_block") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream spill_block failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::spill_block") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::prepare_spill") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream prepare_spill failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::prepare_spill") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::spill_eof") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream spill_eof failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::spill_eof") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_stream::read_next_block") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_stream read_next_block failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_stream::read_next_block") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_sort_sink::sink") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_sort_sink sink failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_sort_sink::sink") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_sort_sink::revoke_memory_cancel") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_sort_sink revoke_memory canceled")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_sort_sink::revoke_memory_cancel") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_sort_sink::revoke_memory_submit_func") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_sort_sink revoke_memory submit_func failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_sort_sink::revoke_memory_submit_func") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_sort_source::recover_spill_data") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_sort_source recover_spill_data failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_sort_source::recover_spill_data") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_sort_source::spill_merged_data") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_sort_source spill_merged_data failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_sort_source::spill_merged_data") + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("fault_inject::spill_sort_source::spill_merged_data") + sql test_sql + } catch(Exception e) { + log.error(e.getMessage()) + assertTrue(e.getMessage().contains("fault_inject spill_sort_source spill_merged_data failed")) + } finally { + GetDebugPoint().disableDebugPointForAllBEs("fault_inject::spill_sort_source::spill_merged_data") + } +} \ No newline at end of file From e7b931b3cbfc55f566dc10cfb62d6ec6835edde2 Mon Sep 17 00:00:00 2001 From: Sun Chenyang Date: Wed, 26 Jun 2024 17:25:17 +0800 Subject: [PATCH 07/31] [refactor](inverted index) Refactor the idx storage format to avoid cyclic references in Thrift files. (#36757) ## Proposed changes `TInvertedIndexStorageFormat` is defined in the `AgentService.thrift`. When other Thirft files use `TInvertedIndexStorageFormat`, need to `include AgentService.thrift`. However, `AgentService.thrift` might already include that file, causing cyclic references. I have defined `TInvetedIndexFileStorageFormat` in `Types.thrift` to represent the inverted index file format. `TInvertedIndexStorageFormat` will be deprecated. This will not impact the upgrade. --- be/src/olap/tablet_meta.cpp | 28 +++++++++++++++---- be/src/olap/tablet_meta.h | 4 +-- .../doris/alter/SchemaChangeHandler.java | 8 ------ .../apache/doris/alter/SchemaChangeJobV2.java | 3 +- .../org/apache/doris/backup/RestoreJob.java | 2 +- .../java/org/apache/doris/catalog/Env.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 14 +++++----- .../apache/doris/catalog/TableProperty.java | 16 +++++------ .../doris/common/util/PropertyAnalyzer.java | 28 +++++++++---------- .../doris/datasource/InternalCatalog.java | 10 +++---- .../apache/doris/master/ReportHandler.java | 4 +-- .../org/apache/doris/qe/ShowExecutor.java | 6 ++-- .../apache/doris/task/CreateReplicaTask.java | 22 +++++++++++---- gensrc/thrift/AgentService.thrift | 4 ++- gensrc/thrift/Types.thrift | 9 ++++++ 15 files changed, 96 insertions(+), 64 deletions(-) diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 26ed8d3ee573c44..cced41a86eee8b6 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -62,6 +62,22 @@ TabletMetaSharedPtr TabletMeta::create( if (request.__isset.binlog_config) { binlog_config = request.binlog_config; } + TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format = + request.inverted_index_file_storage_format; + + // We will discard this format. Don't make any further changes here. + if (request.__isset.inverted_index_storage_format) { + switch (request.inverted_index_storage_format) { + case TInvertedIndexStorageFormat::V1: + inverted_index_file_storage_format = TInvertedIndexFileStorageFormat::V1; + break; + case TInvertedIndexStorageFormat::V2: + inverted_index_file_storage_format = TInvertedIndexFileStorageFormat::V2; + break; + default: + break; + } + } return std::make_shared( request.table_id, request.partition_id, request.tablet_id, request.replica_id, request.tablet_schema.schema_hash, shard_id, request.tablet_schema, next_unique_id, @@ -76,7 +92,7 @@ TabletMetaSharedPtr TabletMeta::create( request.time_series_compaction_file_count_threshold, request.time_series_compaction_time_threshold_seconds, request.time_series_compaction_empty_rowsets_threshold, - request.time_series_compaction_level_threshold, request.inverted_index_storage_format); + request.time_series_compaction_level_threshold, inverted_index_file_storage_format); } TabletMeta::TabletMeta() @@ -97,7 +113,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id int64_t time_series_compaction_time_threshold_seconds, int64_t time_series_compaction_empty_rowsets_threshold, int64_t time_series_compaction_level_threshold, - TInvertedIndexStorageFormat::type inverted_index_storage_format) + TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format) : _tablet_uid(0, 0), _schema(new TabletSchema), _delete_bitmap(new DeleteBitmap(tablet_id)) { @@ -175,15 +191,15 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id break; } - switch (inverted_index_storage_format) { - case TInvertedIndexStorageFormat::V1: + switch (inverted_index_file_storage_format) { + case TInvertedIndexFileStorageFormat::V1: schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); break; - case TInvertedIndexStorageFormat::V2: + case TInvertedIndexFileStorageFormat::V2: schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); break; default: - schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); break; } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index b13bf2f0853918c..32c6fde568c87b2 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -112,8 +112,8 @@ class TabletMeta { int64_t time_series_compaction_time_threshold_seconds = 3600, int64_t time_series_compaction_empty_rowsets_threshold = 5, int64_t time_series_compaction_level_threshold = 1, - TInvertedIndexStorageFormat::type inverted_index_storage_format = - TInvertedIndexStorageFormat::V1); + TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format = + TInvertedIndexFileStorageFormat::V2); // If need add a filed in TableMeta, filed init copy in copy construct function TabletMeta(const TabletMeta& tablet_meta); TabletMeta(TabletMeta&& tablet_meta) = delete; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 8b8d9c1ac72221a..15f8c88bb64231f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -98,7 +98,6 @@ import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.ClearAlterTask; import org.apache.doris.task.UpdateTabletMetaInfoTask; -import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTaskType; @@ -1344,9 +1343,6 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map properties } } - public static TInvertedIndexStorageFormat analyzeInvertedIndexStorageFormat(Map properties) + public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFormat(Map properties) throws AnalysisException { - String invertedIndexStorageFormat = ""; + String invertedIndexFileStorageFormat = ""; if (properties != null && properties.containsKey(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT)) { - invertedIndexStorageFormat = properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT); + invertedIndexFileStorageFormat = properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT); properties.remove(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT); } else { if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { - return TInvertedIndexStorageFormat.V1; + return TInvertedIndexFileStorageFormat.V1; } else { - return TInvertedIndexStorageFormat.V2; + return TInvertedIndexFileStorageFormat.V2; } } - if (invertedIndexStorageFormat.equalsIgnoreCase("v1")) { - return TInvertedIndexStorageFormat.V1; - } else if (invertedIndexStorageFormat.equalsIgnoreCase("v2")) { - return TInvertedIndexStorageFormat.V2; - } else if (invertedIndexStorageFormat.equalsIgnoreCase("default")) { + if (invertedIndexFileStorageFormat.equalsIgnoreCase("v1")) { + return TInvertedIndexFileStorageFormat.V1; + } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v2")) { + return TInvertedIndexFileStorageFormat.V2; + } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default")) { if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { - return TInvertedIndexStorageFormat.V1; + return TInvertedIndexFileStorageFormat.V1; } else { - return TInvertedIndexStorageFormat.V2; + return TInvertedIndexFileStorageFormat.V2; } } else { - throw new AnalysisException("unknown inverted index storage format: " + invertedIndexStorageFormat); + throw new AnalysisException("unknown inverted index storage format: " + invertedIndexFileStorageFormat); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 25d17ad0454e6f9..f3642700c846d30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -166,7 +166,7 @@ import org.apache.doris.task.CreateReplicaTask; import org.apache.doris.task.DropReplicaTask; import org.apache.doris.thrift.TCompressionType; -import org.apache.doris.thrift.TInvertedIndexStorageFormat; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; @@ -2055,7 +2055,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa objectPool); task.setStorageFormat(tbl.getStorageFormat()); - task.setInvertedIndexStorageFormat(tbl.getInvertedIndexStorageFormat()); + task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); task.setClusterKeyIndexes(clusterKeyIndexes); batchTask.addTask(task); // add to AgentTaskQueue for handling finish report. @@ -2410,13 +2410,13 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx } olapTable.setStorageFormat(storageFormat); - TInvertedIndexStorageFormat invertedIndexStorageFormat; + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat; try { - invertedIndexStorageFormat = PropertyAnalyzer.analyzeInvertedIndexStorageFormat(properties); + invertedIndexFileStorageFormat = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(properties); } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } - olapTable.setInvertedIndexStorageFormat(invertedIndexStorageFormat); + olapTable.setInvertedIndexFileStorageFormat(invertedIndexFileStorageFormat); // get compression type TCompressionType compressionType = TCompressionType.LZ4; diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index fa8727a63b902ae..d36144af46cb1ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -889,8 +889,8 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta objectPool); createReplicaTask.setIsRecoverTask(true); - createReplicaTask.setInvertedIndexStorageFormat(olapTable - .getInvertedIndexStorageFormat()); + createReplicaTask.setInvertedIndexFileStorageFormat(olapTable + .getInvertedIndexFileStorageFormat()); createReplicaBatchTask.addTask(createReplicaTask); } else { // just set this replica as bad diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 1af7580b8e2ecca..62e8e6dad5ea5ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -970,13 +970,13 @@ private void handleShowTable() throws AnalysisException { } if (showTableStmt.isVerbose()) { String storageFormat = "NONE"; - String invertedIndexStorageFormat = "NONE"; + String invertedIndexFileStorageFormat = "NONE"; if (tbl instanceof OlapTable) { storageFormat = ((OlapTable) tbl).getStorageFormat().toString(); - invertedIndexStorageFormat = ((OlapTable) tbl).getInvertedIndexStorageFormat().toString(); + invertedIndexFileStorageFormat = ((OlapTable) tbl).getInvertedIndexFileStorageFormat().toString(); } rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(), storageFormat, - invertedIndexStorageFormat)); + invertedIndexFileStorageFormat)); } else { rows.add(Lists.newArrayList(tbl.getName())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 8a658de62bbcca4..022a8be67a970cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -31,6 +31,7 @@ import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TCompressionType; import org.apache.doris.thrift.TCreateTabletReq; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TOlapTableIndex; import org.apache.doris.thrift.TStatusCode; @@ -91,7 +92,7 @@ public class CreateReplicaTask extends AgentTask { // TODO should unify the naming of v1(alpha rowset), v2(beta rowset), it is very confused to read code private TStorageFormat storageFormat = TStorageFormat.V2; - private TInvertedIndexStorageFormat invertedIndexStorageFormat = TInvertedIndexStorageFormat.V1; + private TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat = TInvertedIndexFileStorageFormat.V2; // true if this task is created by recover request(See comment of Config.recover_with_empty_tablet) private boolean isRecoverTask = false; @@ -244,8 +245,8 @@ public void setStorageFormat(TStorageFormat storageFormat) { this.storageFormat = storageFormat; } - public void setInvertedIndexStorageFormat(TInvertedIndexStorageFormat invertedIndexStorageFormat) { - this.invertedIndexStorageFormat = invertedIndexStorageFormat; + public void setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) { + this.invertedIndexFileStorageFormat = invertedIndexFileStorageFormat; } public void setClusterKeyIndexes(List clusterKeyIndexes) { @@ -362,8 +363,19 @@ public TCreateTabletReq toThrift() { createTabletReq.setStorageFormat(storageFormat); } - if (invertedIndexStorageFormat != null) { - createTabletReq.setInvertedIndexStorageFormat(invertedIndexStorageFormat); + if (invertedIndexFileStorageFormat != null) { + createTabletReq.setInvertedIndexFileStorageFormat(invertedIndexFileStorageFormat); + // We will discard the "TInvertedIndexStorageFormat". Don't make any further changes here. + switch (invertedIndexFileStorageFormat) { + case V1: + createTabletReq.setInvertedIndexStorageFormat(TInvertedIndexStorageFormat.V1); + break; + case V2: + createTabletReq.setInvertedIndexStorageFormat(TInvertedIndexStorageFormat.V2); + break; + default: + break; + } } createTabletReq.setTabletType(tabletType); createTabletReq.setCompressionType(compressionType); diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 7d21fb7939e42db..a03cb9df99b69f3 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -180,7 +180,9 @@ struct TCreateTabletReq { 25: optional i64 time_series_compaction_time_threshold_seconds = 3600 26: optional i64 time_series_compaction_empty_rowsets_threshold = 5 27: optional i64 time_series_compaction_level_threshold = 1 - 28: optional TInvertedIndexStorageFormat inverted_index_storage_format = TInvertedIndexStorageFormat.V1 + 28: optional TInvertedIndexStorageFormat inverted_index_storage_format = TInvertedIndexStorageFormat.DEFAULT // Deprecated + 29: optional Types.TInvertedIndexFileStorageFormat inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2 + // For cloud 1000: optional bool is_in_memory = false 1001: optional bool is_persistent = false diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index f13951462a20d52..a13e5c03ce2c26e 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -118,6 +118,15 @@ enum TStorageBackendType { AZURE } +// Enumerates the storage formats for inverted indexes in src_backends. +// This enum is used to distinguish between different organizational methods +// of inverted index data, affecting how the index is stored and accessed. +enum TInvertedIndexFileStorageFormat { + DEFAULT, // Default format, unspecified storage method. + V1, // Index per idx: Each index is stored separately based on its identifier. + V2 // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment. +} + struct TScalarType { 1: required TPrimitiveType type From 9f15d93f7b9b4adf298d484e265b2f1eebf9c955 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Wed, 26 Jun 2024 17:33:20 +0800 Subject: [PATCH 08/31] [fix](pipeline) fix exception safety issue in MultiCastDataStreamer (#36748) ## Proposed changes ```cpp RETURN_IF_ERROR(vectorized::MutableBlock(block).merge(*pos_to_pull->_block)) ``` this line may throw an exception(cannot allocate) --- .../exec/multi_cast_data_streamer.cpp | 20 +------------------ .../pipeline/exec/multi_cast_data_streamer.h | 2 -- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/be/src/pipeline/exec/multi_cast_data_streamer.cpp b/be/src/pipeline/exec/multi_cast_data_streamer.cpp index d3047c42a2ddae9..deebf7d11bb2e2b 100644 --- a/be/src/pipeline/exec/multi_cast_data_streamer.cpp +++ b/be/src/pipeline/exec/multi_cast_data_streamer.cpp @@ -41,9 +41,9 @@ Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* blo pos_to_pull++; _multi_cast_blocks.pop_front(); } else { - pos_to_pull->_used_count--; pos_to_pull->_block->create_same_struct_block(0)->swap(*block); RETURN_IF_ERROR(vectorized::MutableBlock(block).merge(*pos_to_pull->_block)); + pos_to_pull->_used_count--; pos_to_pull++; } } @@ -54,24 +54,6 @@ Status MultiCastDataStreamer::pull(int sender_idx, doris::vectorized::Block* blo return Status::OK(); } -void MultiCastDataStreamer::close_sender(int sender_idx) { - std::lock_guard l(_mutex); - auto& pos_to_pull = _sender_pos_to_read[sender_idx]; - while (pos_to_pull != _multi_cast_blocks.end()) { - if (pos_to_pull->_used_count == 1) { - DCHECK(pos_to_pull == _multi_cast_blocks.begin()); - _cumulative_mem_size -= pos_to_pull->_mem_size; - pos_to_pull++; - _multi_cast_blocks.pop_front(); - } else { - pos_to_pull->_used_count--; - pos_to_pull++; - } - } - _closed_sender_count++; - _block_reading(sender_idx); -} - Status MultiCastDataStreamer::push(RuntimeState* state, doris::vectorized::Block* block, bool eos) { auto rows = block->rows(); COUNTER_UPDATE(_process_rows, rows); diff --git a/be/src/pipeline/exec/multi_cast_data_streamer.h b/be/src/pipeline/exec/multi_cast_data_streamer.h index 0a1276c4f1b0970..2112ebaaf205b12 100644 --- a/be/src/pipeline/exec/multi_cast_data_streamer.h +++ b/be/src/pipeline/exec/multi_cast_data_streamer.h @@ -52,8 +52,6 @@ class MultiCastDataStreamer { Status pull(int sender_idx, vectorized::Block* block, bool* eos); - void close_sender(int sender_idx); - Status push(RuntimeState* state, vectorized::Block* block, bool eos); const RowDescriptor& row_desc() { return _row_desc; } From 0a8eb216ac254e5848ed09d4ecab8f3bff26e9c9 Mon Sep 17 00:00:00 2001 From: wangbo Date: Wed, 26 Jun 2024 17:45:08 +0800 Subject: [PATCH 09/31] [test]add check for query release when p0 finish (#36660) --- .../doris/regression/RegressionTest.groovy | 17 ++++- .../check_before_quit.groovy | 74 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/check_before_quit/check_before_quit.groovy diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy index feafabaddb477e5..20ae8f0e675da0a 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy @@ -173,13 +173,28 @@ class RegressionTest { } } + /* + * support run a specific case after all other cases run. + * if a nonConcurrent case, with name String case_name_check_before_quit = "check_before_quit.groovy", we could make sure it run at last. + */ + String case_name_check_before_quit = "check_before_quit.groovy" + File check_before_quit = null + // 2. collect groovy sources. rootFile.eachFileRecurse { f -> if (f.isFile() && f.name.endsWith('.groovy') && fileFilter.test(f.name) && directoryFilter.test(f.getParent())) { - sources.add(new GroovyFileSource(f)) + if (f.name.equals(case_name_check_before_quit)) { + check_before_quit = f; + } else { + sources.add(new GroovyFileSource(f)) + } } } + if (check_before_quit != null) { + sources.add(new GroovyFileSource(check_before_quit)) + } + return sources } diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy b/regression-test/suites/check_before_quit/check_before_quit.groovy new file mode 100644 index 000000000000000..7623ce64545124b --- /dev/null +++ b/regression-test/suites/check_before_quit/check_before_quit.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("check_before_quit", "nonConcurrent,p0") { + List> be_result = sql "show backends" + List beRow = be_result.get(0) + String beHost = beRow.get(1).toString().trim() + String bePort = beRow.get(4).toString().trim() + logger.info("get be host and port:" + beHost + " " + bePort) + + //NOTE: this suite is used to check whether workload group's query queue works correctly when all query finished + long beginTime = System.currentTimeMillis(); + long timeoutMs = 300 * 1000 // 300s + boolean clear = false + + while ((System.currentTimeMillis() - beginTime) < timeoutMs) { + List> result = sql "show workload groups;" + logger.info("result, ${result}") + + clear = true + for (int i = 0; i < result.size(); i++) { + List row = result.get(i) + int col_size = row.size() + int running_query_num = Integer.valueOf(row.get(col_size - 2).toString()) + int waiting_query_num = Integer.valueOf(row.get(col_size - 1).toString()) + if (running_query_num != 0 || waiting_query_num != 0) { + clear = false + break + } + } + + if (clear) { + break + } + + Thread.sleep(500) + } + + if (clear == false) { + List> result = sql "select QUERY_ID,QUERY_START_TIME,QUERY_TIME_MS,WORKLOAD_GROUP_ID,SQL,QUERY_STATUS from information_schema.active_queries" + logger.info("active queries: ${result}") + if (result.size() == 1) { + clear = true + } + + StringBuilder strBuilder = new StringBuilder() + strBuilder.append("curl http://" + beHost + ":" + bePort + "/api/running_pipeline_tasks/10") + String command = strBuilder.toString() + def process = command.toString().execute() + def code = process.waitFor() + def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + def out = process.getText() + logger.info("Request be running task: code=" + code + ", err=" + err) + logger.info("print be task:${out}") + } + + assertTrue(clear) +} From f413ec422a86a817d4f392cd80a00cc062930fbb Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Wed, 26 Jun 2024 18:14:34 +0800 Subject: [PATCH 10/31] [bug](meta) fix can't deserialize meta from gson about polymorphic function class (#36847) in PR #36552, start to serialize meta by Gson and function class is polymorphic ``` Function.class ---ScalarFunction.class ---AggregateFunction.class ---AliasFunction.class ``` --- .../org/apache/doris/persist/gson/GsonUtils.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 28e7cbc1a13462a..90a1c507011cdd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -61,6 +61,8 @@ import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.RestoreJob; import org.apache.doris.catalog.AggStateType; +import org.apache.doris.catalog.AggregateFunction; +import org.apache.doris.catalog.AliasFunction; import org.apache.doris.catalog.AnyElementType; import org.apache.doris.catalog.AnyStructType; import org.apache.doris.catalog.AnyType; @@ -71,6 +73,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.EsTable; +import org.apache.doris.catalog.Function; import org.apache.doris.catalog.FunctionGenTable; import org.apache.doris.catalog.HMSResource; import org.apache.doris.catalog.HashDistributionInfo; @@ -99,6 +102,7 @@ import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.S3Resource; +import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.catalog.SinglePartitionInfo; @@ -486,6 +490,13 @@ public class GsonUtils { .registerSubtype(FrontendHbResponse.class, FrontendHbResponse.class.getSimpleName()) .registerSubtype(BrokerHbResponse.class, BrokerHbResponse.class.getSimpleName()); + // runtime adapter for class "Function" + private static RuntimeTypeAdapterFactory functionAdapterFactory + = RuntimeTypeAdapterFactory.of(Function.class, "clazz") + .registerSubtype(ScalarFunction.class, ScalarFunction.class.getSimpleName()) + .registerSubtype(AggregateFunction.class, AggregateFunction.class.getSimpleName()) + .registerSubtype(AliasFunction.class, AliasFunction.class.getSimpleName()); + // runtime adapter for class "CloudReplica". private static RuntimeTypeAdapterFactory replicaTypeAdapterFactory = RuntimeTypeAdapterFactory .of(Replica.class, "clazz") @@ -585,6 +596,7 @@ public class GsonUtils { .registerTypeAdapterFactory(partitionTypeAdapterFactory) .registerTypeAdapterFactory(partitionInfoTypeAdapterFactory) .registerTypeAdapterFactory(hbResponseTypeAdapterFactory) + .registerTypeAdapterFactory(functionAdapterFactory) .registerTypeAdapterFactory(rdsTypeAdapterFactory) .registerTypeAdapterFactory(jobExecutorRuntimeTypeAdapterFactory) .registerTypeAdapterFactory(mtmvSnapshotTypeAdapterFactory) From d39b8e1be510ec0257415d24c6df95f5904a6bc7 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 26 Jun 2024 18:56:12 +0800 Subject: [PATCH 11/31] [enhance](mtmv)support partition tvf (#36479) The displayed content is consistent with the show partitions ``` mysql> select * from partitions("catalog"="internal","database"="zd","table"="ss")\G *************************** 1. row *************************** PartitionId: 14004 PartitionName: p1 VisibleVersion: 1 VisibleVersionTime: 2024-06-18 17:41:07 State: NORMAL PartitionKey: k3 Range: [types: [INT]; keys: [1]; ] DistributionKey: k2 Buckets: 2 ReplicationNum: 1 StorageMedium: HDD CooldownTime: 9999-12-31 23:59:59 RemoteStoragePolicy: LastConsistencyCheckTime: \N DataSize: 0.000 IsInMemory: 0 ReplicaAllocation: tag.location.default: 1 IsMutable: 1 SyncWithBaseTables: 1 UnsyncTables: \N 1 row in set (0.02 sec) ``` --- be/src/vec/exec/scan/vmeta_scanner.cpp | 23 ++ be/src/vec/exec/scan/vmeta_scanner.h | 2 + .../catalog/BuiltinTableValuedFunctions.java | 2 + .../doris/common/proc/PartitionsProcDir.java | 77 ++++-- .../functions/table/Partitions.java | 58 +++++ .../visitor/TableValuedFunctionVisitor.java | 5 + .../tablefunction/MetadataGenerator.java | 114 +++++++- .../MetadataTableValuedFunction.java | 2 + .../PartitionsTableValuedFunction.java | 243 ++++++++++++++++++ .../tablefunction/TableValuedFunctionIf.java | 2 + gensrc/thrift/FrontendService.thrift | 1 + gensrc/thrift/PlanNodes.thrift | 8 + gensrc/thrift/Types.thrift | 3 +- .../tvf/test_hms_partitions_tvf.out | 8 + .../tvf/test_partitions_tvf.out | 23 ++ .../tvf/test_hms_partitions_tvf.groovy | 46 ++++ .../tvf/test_partitions_tvf.groovy | 78 ++++++ 17 files changed, 675 insertions(+), 20 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Partitions.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java create mode 100644 regression-test/data/external_table_p0/tvf/test_hms_partitions_tvf.out create mode 100644 regression-test/data/external_table_p0/tvf/test_partitions_tvf.out create mode 100644 regression-test/suites/external_table_p0/tvf/test_hms_partitions_tvf.groovy create mode 100644 regression-test/suites/external_table_p0/tvf/test_partitions_tvf.groovy diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index 44a8624f40585cc..64819cfaaa21129 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -233,6 +233,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) { case TMetadataType::MATERIALIZED_VIEWS: RETURN_IF_ERROR(_build_materialized_views_metadata_request(meta_scan_range, &request)); break; + case TMetadataType::PARTITIONS: + RETURN_IF_ERROR(_build_partitions_metadata_request(meta_scan_range, &request)); + break; case TMetadataType::JOBS: RETURN_IF_ERROR(_build_jobs_metadata_request(meta_scan_range, &request)); break; @@ -401,6 +404,26 @@ Status VMetaScanner::_build_materialized_views_metadata_request( return Status::OK(); } +Status VMetaScanner::_build_partitions_metadata_request(const TMetaScanRange& meta_scan_range, + TFetchSchemaTableDataRequest* request) { + VLOG_CRITICAL << "VMetaScanner::_build_partitions_metadata_request"; + if (!meta_scan_range.__isset.partitions_params) { + return Status::InternalError( + "Can not find TPartitionsMetadataParams from meta_scan_range."); + } + + // create request + request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE); + + // create TMetadataTableRequestParams + TMetadataTableRequestParams metadata_table_params; + metadata_table_params.__set_metadata_type(TMetadataType::PARTITIONS); + metadata_table_params.__set_partitions_metadata_params(meta_scan_range.partitions_params); + + request->__set_metada_table_params(metadata_table_params); + return Status::OK(); +} + Status VMetaScanner::_build_jobs_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) { VLOG_CRITICAL << "VMetaScanner::_build_jobs_metadata_request"; diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index 7bf308cd56a9869..5936130069ca0fe 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -81,6 +81,8 @@ class VMetaScanner : public VScanner { TFetchSchemaTableDataRequest* request); Status _build_materialized_views_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); + Status _build_partitions_metadata_request(const TMetaScanRange& meta_scan_range, + TFetchSchemaTableDataRequest* request); Status _build_jobs_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); Status _build_tasks_metadata_request(const TMetaScanRange& meta_scan_range, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java index 3becd2e102bf815..db66c260e5670ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; +import org.apache.doris.nereids.trees.expressions.functions.table.Partitions; import org.apache.doris.nereids.trees.expressions.functions.table.Query; import org.apache.doris.nereids.trees.expressions.functions.table.S3; import org.apache.doris.nereids.trees.expressions.functions.table.Tasks; @@ -55,6 +56,7 @@ public class BuiltinTableValuedFunctions implements FunctionHelper { tableValued(Numbers.class, "numbers"), tableValued(S3.class, "s3"), tableValued(MvInfos.class, "mv_infos"), + tableValued(Partitions.class, "partitions"), tableValued(Jobs.class, "jobs"), tableValued(Tasks.class, "tasks"), tableValued(Query.class, "query") diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index b36bc5eb022ffec..48e406b66d906b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -46,6 +46,8 @@ import org.apache.doris.common.util.OrderByPair; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.mtmv.MTMVPartitionUtil; +import org.apache.doris.thrift.TCell; +import org.apache.doris.thrift.TRow; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -222,12 +224,22 @@ public BaseProcResult getBasicProcResult(List> partitionInfos) } private List> getPartitionInfos() throws AnalysisException { + List, TRow>> partitionInfosInrernal = getPartitionInfosInrernal(); + return partitionInfosInrernal.stream().map(pair -> pair.first).collect(Collectors.toList()); + } + + public List getPartitionInfosForTvf() throws AnalysisException { + List, TRow>> partitionInfosInrernal = getPartitionInfosInrernal(); + return partitionInfosInrernal.stream().map(pair -> pair.second).collect(Collectors.toList()); + } + + private List, TRow>> getPartitionInfosInrernal() throws AnalysisException { Preconditions.checkNotNull(db); Preconditions.checkNotNull(olapTable); Preconditions.checkState(olapTable.isManagedTable()); // get info - List> partitionInfos = new ArrayList>(); + List, TRow>> partitionInfos = new ArrayList, TRow>>(); olapTable.readLock(); try { List partitionIds; @@ -259,13 +271,19 @@ private List> getPartitionInfos() throws AnalysisException { Partition partition = olapTable.getPartition(partitionId); List partitionInfo = new ArrayList(); + TRow trow = new TRow(); String partitionName = partition.getName(); partitionInfo.add(partitionId); + trow.addToColumnValue(new TCell().setLongVal(partitionId)); partitionInfo.add(partitionName); + trow.addToColumnValue(new TCell().setStringVal(partitionName)); partitionInfo.add(partition.getVisibleVersion()); - partitionInfo.add(TimeUtils.longToTimeString(partition.getVisibleVersionTime())); + trow.addToColumnValue(new TCell().setLongVal(partition.getVisibleVersion())); + String visibleTime = TimeUtils.longToTimeString(partition.getVisibleVersionTime()); + partitionInfo.add(visibleTime); + trow.addToColumnValue(new TCell().setStringVal(visibleTime)); partitionInfo.add(partition.getState()); - + trow.addToColumnValue(new TCell().setStringVal(partition.getState().toString())); if (tblPartitionInfo.getType() == PartitionType.RANGE || tblPartitionInfo.getType() == PartitionType.LIST) { List partitionColumns = tblPartitionInfo.getPartitionColumns(); @@ -273,11 +291,17 @@ private List> getPartitionInfos() throws AnalysisException { for (Column column : partitionColumns) { colNames.add(column.getName()); } - partitionInfo.add(joiner.join(colNames)); - partitionInfo.add(tblPartitionInfo.getItem(partitionId).getItems().toString()); + String colNamesStr = joiner.join(colNames); + partitionInfo.add(colNamesStr); + trow.addToColumnValue(new TCell().setStringVal(colNamesStr)); + String itemStr = tblPartitionInfo.getItem(partitionId).getItems().toString(); + partitionInfo.add(itemStr); + trow.addToColumnValue(new TCell().setStringVal(itemStr)); } else { partitionInfo.add(""); + trow.addToColumnValue(new TCell().setStringVal("")); partitionInfo.add(""); + trow.addToColumnValue(new TCell().setStringVal("")); } // distribution @@ -293,47 +317,70 @@ private List> getPartitionInfos() throws AnalysisException { sb.append(distributionColumns.get(i).getName()); } partitionInfo.add(sb.toString()); + trow.addToColumnValue(new TCell().setStringVal(sb.toString())); } else { partitionInfo.add("RANDOM"); + trow.addToColumnValue(new TCell().setStringVal("RANDOM")); } partitionInfo.add(distributionInfo.getBucketNum()); + trow.addToColumnValue(new TCell().setIntVal(distributionInfo.getBucketNum())); // replica num - partitionInfo.add(tblPartitionInfo.getReplicaAllocation(partitionId).getTotalReplicaNum()); + short totalReplicaNum = tblPartitionInfo.getReplicaAllocation(partitionId).getTotalReplicaNum(); + partitionInfo.add(totalReplicaNum); + trow.addToColumnValue(new TCell().setIntVal(totalReplicaNum)); DataProperty dataProperty = tblPartitionInfo.getDataProperty(partitionId); partitionInfo.add(dataProperty.getStorageMedium().name()); - partitionInfo.add(TimeUtils.longToTimeString(dataProperty.getCooldownTimeMs())); + trow.addToColumnValue(new TCell().setStringVal(dataProperty.getStorageMedium().name())); + String cooldownTimeStr = TimeUtils.longToTimeString(dataProperty.getCooldownTimeMs()); + partitionInfo.add(cooldownTimeStr); + trow.addToColumnValue(new TCell().setStringVal(cooldownTimeStr)); partitionInfo.add(dataProperty.getStoragePolicy()); - - partitionInfo.add(TimeUtils.longToTimeString(partition.getLastCheckTime())); - + trow.addToColumnValue(new TCell().setStringVal(dataProperty.getStoragePolicy())); + String lastCheckTime = TimeUtils.longToTimeString(partition.getLastCheckTime()); + partitionInfo.add(lastCheckTime); + trow.addToColumnValue(new TCell().setStringVal(lastCheckTime)); long dataSize = partition.getDataSize(false); Pair sizePair = DebugUtil.getByteUint(dataSize); String readableSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " " + sizePair.second; partitionInfo.add(readableSize); - partitionInfo.add(tblPartitionInfo.getIsInMemory(partitionId)); + trow.addToColumnValue(new TCell().setStringVal(readableSize)); + boolean isInMemory = tblPartitionInfo.getIsInMemory(partitionId); + partitionInfo.add(isInMemory); + trow.addToColumnValue(new TCell().setBoolVal(isInMemory)); // replica allocation - partitionInfo.add(tblPartitionInfo.getReplicaAllocation(partitionId).toCreateStmt()); + String replica = tblPartitionInfo.getReplicaAllocation(partitionId).toCreateStmt(); + partitionInfo.add(replica); + trow.addToColumnValue(new TCell().setStringVal(replica)); - partitionInfo.add(tblPartitionInfo.getIsMutable(partitionId)); + boolean isMutable = tblPartitionInfo.getIsMutable(partitionId); + partitionInfo.add(isMutable); + trow.addToColumnValue(new TCell().setBoolVal(isMutable)); if (olapTable instanceof MTMV) { if (StringUtils.isEmpty(mtmvPartitionSyncErrorMsg)) { List partitionUnSyncTables = partitionsUnSyncTables.getOrDefault(partitionId, Lists.newArrayList()); - partitionInfo.add(CollectionUtils.isEmpty(partitionUnSyncTables)); + boolean isSync = CollectionUtils.isEmpty(partitionUnSyncTables); + partitionInfo.add(isSync); + trow.addToColumnValue(new TCell().setBoolVal(isSync)); partitionInfo.add(partitionUnSyncTables.toString()); + trow.addToColumnValue(new TCell().setStringVal(partitionUnSyncTables.toString())); } else { partitionInfo.add(false); + trow.addToColumnValue(new TCell().setBoolVal(false)); partitionInfo.add(mtmvPartitionSyncErrorMsg); + trow.addToColumnValue(new TCell().setStringVal(mtmvPartitionSyncErrorMsg)); } } else { partitionInfo.add(true); + trow.addToColumnValue(new TCell().setBoolVal(true)); partitionInfo.add(FeConstants.null_string); + trow.addToColumnValue(new TCell().setStringVal(FeConstants.null_string)); } - partitionInfos.add(partitionInfo); + partitionInfos.add(Pair.of(partitionInfo, trow)); } } finally { olapTable.readUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Partitions.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Partitions.java new file mode 100644 index 000000000000000..6dcd31a9651b869 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Partitions.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.table; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.tablefunction.PartitionsTableValuedFunction; +import org.apache.doris.tablefunction.TableValuedFunctionIf; + +import java.util.Map; + +/** + * partitions + */ +public class Partitions extends TableValuedFunction { + public Partitions(Properties properties) { + super("partitions", properties); + } + + @Override + public FunctionSignature customSignature() { + return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, getArgumentsTypes()); + } + + @Override + protected TableValuedFunctionIf toCatalogFunction() { + try { + Map arguments = getTVFProperties().getMap(); + return new PartitionsTableValuedFunction(arguments); + } catch (Throwable t) { + throw new AnalysisException("Can not build PartitionsTableValuedFunction by " + + this + ": " + t.getMessage(), t); + } + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPartitions(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java index bd09a81b0114f42..ca14edd87def394 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; +import org.apache.doris.nereids.trees.expressions.functions.table.Partitions; import org.apache.doris.nereids.trees.expressions.functions.table.Query; import org.apache.doris.nereids.trees.expressions.functions.table.S3; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; @@ -54,6 +55,10 @@ default R visitMvInfos(MvInfos mvInfos, C context) { return visitTableValuedFunction(mvInfos, context); } + default R visitPartitions(Partitions partitions, C context) { + return visitTableValuedFunction(partitions, context); + } + default R visitJobs(Jobs jobs, C context) { return visitTableValuedFunction(jobs, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index b3ed195a935c990..91690e23b955160 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; @@ -35,11 +36,14 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.proc.FrontendsProcNode; +import org.apache.doris.common.proc.PartitionsProcDir; import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergMetadataCache; +import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.job.common.JobType; import org.apache.doris.job.extensions.mtmv.MTMVJob; import org.apache.doris.job.task.AbstractTask; @@ -67,6 +71,7 @@ import org.apache.doris.thrift.TMetadataTableRequestParams; import org.apache.doris.thrift.TMetadataType; import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPartitionsMetadataParams; import org.apache.doris.thrift.TPipelineWorkloadGroup; import org.apache.doris.thrift.TRow; import org.apache.doris.thrift.TSchemaTableRequestParams; @@ -175,6 +180,9 @@ public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableData case MATERIALIZED_VIEWS: result = mtmvMetadataResult(params); break; + case PARTITIONS: + result = partitionMetadataResult(params); + break; case JOBS: result = jobMetadataResult(params); break; @@ -775,6 +783,104 @@ private static TFetchSchemaTableDataResult mtmvMetadataResult(TMetadataTableRequ return result; } + private static TFetchSchemaTableDataResult partitionMetadataResult(TMetadataTableRequestParams params) { + if (LOG.isDebugEnabled()) { + LOG.debug("partitionMetadataResult() start"); + } + if (!params.isSetPartitionsMetadataParams()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Partitions metadata params is not set."); + } + return errorResult("Partitions metadata params is not set."); + } + + TPartitionsMetadataParams partitionsMetadataParams = params.getPartitionsMetadataParams(); + String catalogName = partitionsMetadataParams.getCatalog(); + if (LOG.isDebugEnabled()) { + LOG.debug("catalogName: " + catalogName); + } + String dbName = partitionsMetadataParams.getDatabase(); + if (LOG.isDebugEnabled()) { + LOG.debug("dbName: " + dbName); + } + String tableName = partitionsMetadataParams.getTable(); + if (LOG.isDebugEnabled()) { + LOG.debug("tableName: " + tableName); + } + + CatalogIf catalog; + TableIf table; + DatabaseIf db; + try { + catalog = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrAnalysisException(catalogName); + db = catalog.getDbOrAnalysisException(dbName); + table = db.getTableOrAnalysisException(tableName); + } catch (AnalysisException e) { + LOG.warn(e.getMessage()); + return errorResult(e.getMessage()); + } + + if (catalog instanceof InternalCatalog) { + return dealInternalCatalog((Database) db, table); + } else if (catalog instanceof MaxComputeExternalCatalog) { + return dealMaxComputeCatalog((MaxComputeExternalCatalog) catalog, dbName, tableName); + } else if (catalog instanceof HMSExternalCatalog) { + return dealHMSCatalog((HMSExternalCatalog) catalog, dbName, tableName); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("partitionMetadataResult() end"); + } + return errorResult("not support catalog: " + catalogName); + } + + private static TFetchSchemaTableDataResult dealHMSCatalog(HMSExternalCatalog catalog, String dbName, + String tableName) { + List dataBatch = Lists.newArrayList(); + List partitionNames = catalog.getClient().listPartitionNames(dbName, tableName); + for (String partition : partitionNames) { + TRow trow = new TRow(); + trow.addToColumnValue(new TCell().setStringVal(partition)); + dataBatch.add(trow); + } + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + + private static TFetchSchemaTableDataResult dealMaxComputeCatalog(MaxComputeExternalCatalog catalog, String dbName, + String tableName) { + List dataBatch = Lists.newArrayList(); + List partitionNames = catalog.listPartitionNames(dbName, tableName); + for (String partition : partitionNames) { + TRow trow = new TRow(); + trow.addToColumnValue(new TCell().setStringVal(partition)); + dataBatch.add(trow); + } + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + + private static TFetchSchemaTableDataResult dealInternalCatalog(Database db, TableIf table) { + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + if (!(table instanceof OlapTable)) { + return errorResult("not olap table"); + } + PartitionsProcDir dir = new PartitionsProcDir(db, (OlapTable) table, false); + try { + List dataBatch = dir.getPartitionInfosForTvf(); + result.setDataBatch(dataBatch); + } catch (AnalysisException e) { + return errorResult(e.getMessage()); + } + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + private static TFetchSchemaTableDataResult jobMetadataResult(TMetadataTableRequestParams params) { if (!params.isSetJobsMetadataParams()) { return errorResult("Jobs metadata params is not set."); @@ -917,9 +1023,9 @@ private static TFetchSchemaTableDataResult tableOptionsMetadataResult(TSchemaTab trow.addToColumnValue(new TCell().setStringVal(catalog.getName())); // TABLE_CATALOG trow.addToColumnValue(new TCell().setStringVal(database.getFullName())); // TABLE_SCHEMA trow.addToColumnValue( - new TCell().setStringVal(olapTable.getKeysType().toMetadata())); //TABLE_MODEL + new TCell().setStringVal(olapTable.getKeysType().toMetadata())); // TABLE_MODEL trow.addToColumnValue( - new TCell().setStringVal(olapTable.getKeyColAsString())); // key columTypes + new TCell().setStringVal(olapTable.getKeyColAsString())); // key columTypes DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); if (distributionInfo.getType() == DistributionInfoType.HASH) { @@ -936,7 +1042,7 @@ private static TFetchSchemaTableDataResult tableOptionsMetadataResult(TSchemaTab trow.addToColumnValue(new TCell().setStringVal("")); } else { trow.addToColumnValue( - new TCell().setStringVal(distributeKey.toString())); + new TCell().setStringVal(distributeKey.toString())); } trow.addToColumnValue(new TCell().setStringVal("HASH")); // DISTRIBUTE_TYPE } else { @@ -951,7 +1057,7 @@ private static TFetchSchemaTableDataResult tableOptionsMetadataResult(TSchemaTab } else { try { trow.addToColumnValue( - new TCell().setStringVal(property.getPropertiesString())); // PROPERTIES + new TCell().setStringVal(property.getPropertiesString())); // PROPERTIES } catch (IOException e) { return errorResult(e.getMessage()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java index af37bcd10e4c15a..a7e25bc7f824453 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java @@ -44,6 +44,8 @@ public static Integer getColumnIndexFromColumnName(TMetadataType type, String co return CatalogsTableValuedFunction.getColumnIndexFromColumnName(columnName); case MATERIALIZED_VIEWS: return MvInfosTableValuedFunction.getColumnIndexFromColumnName(columnName); + case PARTITIONS: + return PartitionsTableValuedFunction.getColumnIndexFromColumnName(columnName, params); case JOBS: return JobsTableValuedFunction.getColumnIndexFromColumnName(columnName, params); case TASKS: diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java new file mode 100644 index 000000000000000..1ceddeb89cf4a88 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java @@ -0,0 +1,243 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; +import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TMetaScanRange; +import org.apache.doris.thrift.TMetadataTableRequestParams; +import org.apache.doris.thrift.TMetadataType; +import org.apache.doris.thrift.TPartitionsMetadataParams; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The Implement of table valued function + * partitions("database" = "db1","table" = "table1"). + */ +public class PartitionsTableValuedFunction extends MetadataTableValuedFunction { + private static final Logger LOG = LogManager.getLogger(PartitionsTableValuedFunction.class); + + public static final String NAME = "partitions"; + + private static final String CATALOG = "catalog"; + private static final String DB = "database"; + private static final String TABLE = "table"; + + private static final ImmutableSet PROPERTIES_SET = ImmutableSet.of(CATALOG, DB, TABLE); + + private static final ImmutableList SCHEMA_FOR_OLAP_TABLE = ImmutableList.of( + new Column("PartitionId", ScalarType.createType(PrimitiveType.BIGINT)), + new Column("PartitionName", ScalarType.createStringType()), + new Column("VisibleVersion", ScalarType.createType(PrimitiveType.BIGINT)), + new Column("VisibleVersionTime", ScalarType.createStringType()), + new Column("State", ScalarType.createStringType()), + new Column("PartitionKey", ScalarType.createStringType()), + new Column("Range", ScalarType.createStringType()), + new Column("DistributionKey", ScalarType.createStringType()), + new Column("Buckets", ScalarType.createType(PrimitiveType.INT)), + new Column("ReplicationNum", ScalarType.createType(PrimitiveType.INT)), + new Column("StorageMedium", ScalarType.createStringType()), + new Column("CooldownTime", ScalarType.createStringType()), + new Column("RemoteStoragePolicy", ScalarType.createStringType()), + new Column("LastConsistencyCheckTime", ScalarType.createStringType()), + new Column("DataSize", ScalarType.createStringType()), + new Column("IsInMemory", ScalarType.createType(PrimitiveType.BOOLEAN)), + new Column("ReplicaAllocation", ScalarType.createStringType()), + new Column("IsMutable", ScalarType.createType(PrimitiveType.BOOLEAN)), + new Column("SyncWithBaseTables", ScalarType.createType(PrimitiveType.BOOLEAN)), + new Column("UnsyncTables", ScalarType.createStringType())); + + private static final ImmutableList SCHEMA_FOR_EXTERNAL_TABLE = ImmutableList.of( + new Column("Partition", ScalarType.createStringType())); + + private static final ImmutableMap OLAP_TABLE_COLUMN_TO_INDEX; + private static final ImmutableMap EXTERNAL_TABLE_COLUMN_TO_INDEX; + + static { + ImmutableMap.Builder builder = new ImmutableMap.Builder(); + for (int i = 0; i < SCHEMA_FOR_OLAP_TABLE.size(); i++) { + builder.put(SCHEMA_FOR_OLAP_TABLE.get(i).getName().toLowerCase(), i); + } + OLAP_TABLE_COLUMN_TO_INDEX = builder.build(); + + ImmutableMap.Builder otherBuilder = new ImmutableMap.Builder(); + for (int i = 0; i < SCHEMA_FOR_EXTERNAL_TABLE.size(); i++) { + otherBuilder.put(SCHEMA_FOR_EXTERNAL_TABLE.get(i).getName().toLowerCase(), i); + } + EXTERNAL_TABLE_COLUMN_TO_INDEX = otherBuilder.build(); + } + + public static Integer getColumnIndexFromColumnName(String columnName, TMetadataTableRequestParams params) + throws org.apache.doris.common.AnalysisException { + if (!params.isSetPartitionsMetadataParams()) { + throw new org.apache.doris.common.AnalysisException("Partitions metadata params is not set."); + } + TPartitionsMetadataParams partitionsMetadataParams = params.getPartitionsMetadataParams(); + String catalogName = partitionsMetadataParams.getCatalog(); + if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalogName)) { + return OLAP_TABLE_COLUMN_TO_INDEX.get(columnName.toLowerCase()); + } else { + return EXTERNAL_TABLE_COLUMN_TO_INDEX.get(columnName.toLowerCase()); + } + } + + private final String catalogName; + private final String databaseName; + private final String tableName; + + public PartitionsTableValuedFunction(Map params) throws AnalysisException { + if (LOG.isDebugEnabled()) { + LOG.debug("PartitionsTableValuedFunction() start"); + } + Map validParams = Maps.newHashMap(); + for (String key : params.keySet()) { + if (!PROPERTIES_SET.contains(key.toLowerCase())) { + throw new AnalysisException("'" + key + "' is invalid property"); + } + // check ctl, db, tbl + validParams.put(key.toLowerCase(), params.get(key)); + } + String catalogName = validParams.get(CATALOG); + String dbName = validParams.get(DB); + String tableName = validParams.get(TABLE); + if (StringUtils.isEmpty(catalogName) || StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tableName)) { + throw new AnalysisException("Invalid partitions metadata query"); + } + analyze(catalogName, dbName, tableName); + this.catalogName = catalogName; + this.databaseName = dbName; + this.tableName = tableName; + if (LOG.isDebugEnabled()) { + LOG.debug("PartitionsTableValuedFunction() end"); + } + } + + private void analyze(String catalogName, String dbName, String tableName) { + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), catalogName, dbName, + tableName, PrivPredicate.SHOW)) { + String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SHOW PARTITIONS", + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), + catalogName + ": " + dbName + ": " + tableName); + throw new AnalysisException(message); + } + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + if (catalog == null) { + throw new AnalysisException("can not find catalog: " + catalogName); + } + // disallow unsupported catalog + if (!(catalog.isInternalCatalog() || catalog instanceof HMSExternalCatalog + || catalog instanceof MaxComputeExternalCatalog)) { + throw new AnalysisException(String.format("Catalog of type '%s' is not allowed in ShowPartitionsStmt", + catalog.getType())); + } + + Optional db = catalog.getDb(dbName); + if (!db.isPresent()) { + throw new AnalysisException("can not find database: " + dbName); + } + TableIf table = null; + try { + table = db.get().getTableOrMetaException(tableName, TableType.OLAP, + TableType.HMS_EXTERNAL_TABLE, TableType.MAX_COMPUTE_EXTERNAL_TABLE); + } catch (MetaNotFoundException e) { + throw new AnalysisException(e.getMessage(), e); + } + + if (table instanceof HMSExternalTable) { + if (((HMSExternalTable) table).isView()) { + throw new AnalysisException("Table " + tableName + " is not a partitioned table"); + } + if (CollectionUtils.isEmpty(((HMSExternalTable) table).getPartitionColumns())) { + throw new AnalysisException("Table " + tableName + " is not a partitioned table"); + } + return; + } + + if (table instanceof MaxComputeExternalTable) { + if (((MaxComputeExternalTable) table).getOdpsTable().getPartitions().isEmpty()) { + throw new AnalysisException("Table " + tableName + " is not a partitioned table"); + } + } + } + + @Override + public TMetadataType getMetadataType() { + return TMetadataType.PARTITIONS; + } + + @Override + public TMetaScanRange getMetaScanRange() { + if (LOG.isDebugEnabled()) { + LOG.debug("getMetaScanRange() start"); + } + TMetaScanRange metaScanRange = new TMetaScanRange(); + metaScanRange.setMetadataType(TMetadataType.PARTITIONS); + TPartitionsMetadataParams partitionParam = new TPartitionsMetadataParams(); + partitionParam.setCatalog(catalogName); + partitionParam.setDatabase(databaseName); + partitionParam.setTable(tableName); + metaScanRange.setPartitionsParams(partitionParam); + if (LOG.isDebugEnabled()) { + LOG.debug("getMetaScanRange() end"); + } + return metaScanRange; + } + + @Override + public String getTableName() { + return "PartitionsTableValuedFunction"; + } + + @Override + public List getTableColumns() throws AnalysisException { + if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalogName)) { + return SCHEMA_FOR_OLAP_TABLE; + } else { + return SCHEMA_FOR_EXTERNAL_TABLE; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index c99da94de0fe9e5..5d9e807c11f1ed0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -66,6 +66,8 @@ public static TableValuedFunctionIf getTableFunction(String funcName, Map> res = sql """ select * from partitions('catalog'='internal',"database"="${dbName}","table"="${tableName}"); """ + logger.info("res: " + res.toString()) + + assertEquals(1, res.size()); + // PartitionName + assertEquals("p1", res[0][1]); + // State + assertEquals("NORMAL", res[0][4]); + // PartitionKey + assertEquals("k3", res[0][5]); + // Buckets + assertEquals(2, res[0][8]); + // ReplicationNum + assertEquals(1, res[0][9]); + // StorageMedium + assertEquals("HDD", res[0][10]); + // ReplicaAllocation + assertEquals("tag.location.default: 1", res[0][16]); + // IsMutable + assertEquals(true, res[0][17]); + // SyncWithBaseTables + assertEquals(true, res[0][18]); + + + // test exception + test { + sql """ select * from partitions("catalog"="internal","database"="${dbName}","table"="xxx"); """ + // check exception + exception "xxx" + } + test { + sql """ select * from partitions("database"="${dbName}"); """ + // check exception + exception "Invalid" + } + + sql """drop table if exists `${tableName}`""" +} From 3a8dbe48c3e3a94e766a9d551aba2344d36bd429 Mon Sep 17 00:00:00 2001 From: Peyz <30936555+iszhangpch@users.noreply.github.com> Date: Wed, 26 Jun 2024 19:02:34 +0800 Subject: [PATCH 12/31] [improvement](meta) Switch meta serialization to gson 4 (#36568) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Proposed changes Issue Number: Switch meta serialization to gson Contains the following classes： RoutineLoadJob UserPropertyInfo AnalyzeDeletionLog CreateTableInfo HbPackage ~~PartitionPersistInfo~~ ReplicaPersistInfo RoutineLoadOperation --------- Co-authored-by: zhangpeicheng Co-authored-by: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com> --- .../apache/doris/common/FeMetaVersion.java | 4 +- .../doris/analysis/ImportColumnDesc.java | 3 + .../apache/doris/analysis/StorageBackend.java | 2 + .../org/apache/doris/backup/Repository.java | 48 +++++-- .../apache/doris/backup/RepositoryMgr.java | 36 ++++-- .../apache/doris/fs/PersistentFileSystem.java | 28 ++-- .../load/loadv2/LoadJobFinalOperation.java | 28 +--- .../loadv2/MiniLoadTxnCommitAttachment.java | 16 +-- .../doris/load/routineload/KafkaProgress.java | 48 ++++--- .../load/routineload/KafkaRoutineLoadJob.java | 25 +--- .../RLTaskTxnCommitAttachment.java | 13 +- .../load/routineload/RoutineLoadJob.java | 120 ++++++++++-------- .../load/routineload/RoutineLoadProgress.java | 11 +- .../routineload/RoutineLoadStatistic.java | 10 +- .../mysql/privilege/UserPropertyInfo.java | 32 +++-- .../doris/persist/AnalyzeDeletionLog.java | 17 ++- .../apache/doris/persist/CreateTableInfo.java | 17 ++- .../org/apache/doris/persist/HbPackage.java | 22 +++- .../doris/persist/PartitionPersistInfo.java | 1 + .../doris/persist/ReplicaPersistInfo.java | 65 ++++++---- .../doris/persist/RoutineLoadOperation.java | 21 ++- .../persist/gson/GsonPreProcessable.java | 24 ++++ .../apache/doris/persist/gson/GsonUtils.java | 39 +++++- .../org/apache/doris/qe/OriginStatement.java | 10 +- .../org/apache/doris/task/LoadTaskInfo.java | 3 + .../doris/transaction/TransactionState.java | 1 + .../transaction/TxnCommitAttachment.java | 48 ++++--- .../RoutineLoadTaskSchedulerTest.java | 3 +- .../transaction/GlobalTransactionMgrTest.java | 5 +- 29 files changed, 407 insertions(+), 293 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonPreProcessable.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 2c1f22735ffbee5..7aa98625639636f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -94,9 +94,11 @@ public final class FeMetaVersion { public static final int VERSION_135 = 135; // For mate gson public static final int VERSION_136 = 136; + // For mate gson + public static final int VERSION_137 = 137; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_136; + public static final int VERSION_CURRENT = VERSION_137; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java index 13040bdcb5cf89d..e62fca2c27f7ad1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java @@ -21,9 +21,12 @@ import org.apache.doris.catalog.Column; import com.google.common.base.Preconditions; +import com.google.gson.annotations.SerializedName; public class ImportColumnDesc { + @SerializedName("cn") private String columnName; + @SerializedName("expr") private Expr expr; public ImportColumnDesc(ImportColumnDesc other) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java index 5287118a5867613..b07725d2507175d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java @@ -27,6 +27,7 @@ import org.apache.doris.thrift.TStorageBackendType; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import org.apache.commons.lang3.StringUtils; import java.util.Map; @@ -153,6 +154,7 @@ public enum StorageType { STREAM("Stream load pipe"), AZURE("MicroSoft Azure Blob"); + @SerializedName("desc") private final String description; StorageType(String description) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 5a7b0d044fba002..b571393efbbd81b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -24,12 +24,14 @@ import org.apache.doris.catalog.FsBroker; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.PersistentFileSystem; import org.apache.doris.fs.remote.AzureFileSystem; import org.apache.doris.fs.remote.BrokerFileSystem; @@ -37,12 +39,15 @@ import org.apache.doris.fs.remote.RemoteFileSystem; import org.apache.doris.fs.remote.S3FileSystem; import org.apache.doris.fs.remote.dfs.DFSFileSystem; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.system.Backend; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -91,7 +96,7 @@ * * __10023_seg2.dat.DNW231dnklawd * * __10023.hdr.dnmwDDWI92dDko */ -public class Repository implements Writable { +public class Repository implements Writable, GsonPostProcessable { public static final String PREFIX_REPO = "__palo_repository_"; public static final String PREFIX_SNAPSHOT_DIR = "__ss_"; public static final String PREFIX_DB = "__db_"; @@ -110,19 +115,25 @@ public class Repository implements Writable { private static final String PATH_DELIMITER = "/"; private static final String CHECKSUM_SEPARATOR = "."; + @SerializedName("id") private long id; + @SerializedName("n") private String name; private String errMsg; + @SerializedName("ct") private long createTime; // If True, user can not backup data to this repo. + @SerializedName("iro") private boolean isReadOnly; // BOS location should start with "bos://your_bucket_name/" // and the specified bucket should exist. + @SerializedName("lo") private String location; - private RemoteFileSystem fileSystem; + @SerializedName("fs") + private PersistentFileSystem fileSystem; private Repository() { // for persist @@ -184,9 +195,26 @@ public static String replaceFileNameWithChecksumFileName(String origPath, String } public static Repository read(DataInput in) throws IOException { - Repository repo = new Repository(); - repo.readFields(in); - return repo; + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + Repository repo = new Repository(); + repo.readFields(in); + return repo; + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), Repository.class); + } + } + + @Override + public void gsonPostProcess() { + StorageBackend.StorageType type = StorageBackend.StorageType.BROKER; + if (this.fileSystem.properties.containsKey(PersistentFileSystem.STORAGE_TYPE)) { + type = StorageBackend.StorageType.valueOf( + this.fileSystem.properties.get(PersistentFileSystem.STORAGE_TYPE)); + this.fileSystem.properties.remove(PersistentFileSystem.STORAGE_TYPE); + } + this.fileSystem = FileSystemFactory.get(this.fileSystem.getName(), + type, + this.fileSystem.getProperties()); } public long getId() { @@ -209,7 +237,7 @@ public String getErrorMsg() { return errMsg; } - public RemoteFileSystem getRemoteFileSystem() { + public PersistentFileSystem getRemoteFileSystem() { return fileSystem; } @@ -836,14 +864,10 @@ private String allocLocalFileSuffix() { @Override public void write(DataOutput out) throws IOException { - out.writeLong(id); - Text.writeString(out, name); - out.writeBoolean(isReadOnly); - Text.writeString(out, location); - fileSystem.write(out); - out.writeLong(createTime); + Text.writeString(out, GsonUtils.GSON.toJson(this)); } + @Deprecated public void readFields(DataInput in) throws IOException { id = in.readLong(); name = Text.readString(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java index 0216779dcd39b36..853c18414490467 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java @@ -19,13 +19,18 @@ import org.apache.doris.backup.Status.ErrCode; import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Daemon; import org.apache.doris.fs.remote.AzureFileSystem; import org.apache.doris.fs.remote.S3FileSystem; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -33,18 +38,19 @@ import java.io.DataOutput; import java.io.IOException; import java.util.List; -import java.util.Map; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; /* * A manager to manage all backup repositories */ -public class RepositoryMgr extends Daemon implements Writable { +public class RepositoryMgr extends Daemon implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RepositoryMgr.class); // all key should be in lower case - private Map repoNameMap = Maps.newConcurrentMap(); - private Map repoIdMap = Maps.newConcurrentMap(); + @SerializedName("rn") + private ConcurrentMap repoNameMap = Maps.newConcurrentMap(); + private ConcurrentMap repoIdMap = Maps.newConcurrentMap(); private ReentrantLock lock = new ReentrantLock(); @@ -153,19 +159,27 @@ public List> getReposInfo() { } public static RepositoryMgr read(DataInput in) throws IOException { - RepositoryMgr mgr = new RepositoryMgr(); - mgr.readFields(in); - return mgr; + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + RepositoryMgr mgr = new RepositoryMgr(); + mgr.readFields(in); + return mgr; + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), RepositoryMgr.class); + } + } @Override public void write(DataOutput out) throws IOException { - out.writeInt(repoNameMap.size()); - for (Repository repo : repoNameMap.values()) { - repo.write(out); - } + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + @Override + public void gsonPostProcess() { + repoNameMap.forEach((n, repo) -> repoIdMap.put(repo.getId(), repo)); } + @Deprecated public void readFields(DataInput in) throws IOException { int size = in.readInt(); for (int i = 0; i < size; i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java index d2cc898766d980c..9cb156036d93041 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java @@ -19,24 +19,25 @@ import org.apache.doris.analysis.StorageBackend; import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.fs.remote.RemoteFileSystem; +import org.apache.doris.persist.gson.GsonPreProcessable; import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; import java.util.Map; /** * Use for persistence, Repository will persist properties of file system. */ -public abstract class PersistentFileSystem implements FileSystem, Writable { +public abstract class PersistentFileSystem implements FileSystem, GsonPreProcessable { public static final String STORAGE_TYPE = "_DORIS_STORAGE_TYPE_"; - protected Map properties = Maps.newHashMap(); - protected String name; - protected StorageBackend.StorageType type; + @SerializedName("prop") + public Map properties = Maps.newHashMap(); + @SerializedName("n") + public String name; + public StorageBackend.StorageType type; public boolean needFullPath() { return type == StorageBackend.StorageType.S3 @@ -66,7 +67,8 @@ public StorageBackend.StorageType getStorageType() { * @param in persisted data * @return file systerm */ - public static RemoteFileSystem read(DataInput in) throws IOException { + @Deprecated + public static PersistentFileSystem read(DataInput in) throws IOException { String name = Text.readString(in); Map properties = Maps.newHashMap(); StorageBackend.StorageType type = StorageBackend.StorageType.BROKER; @@ -83,14 +85,8 @@ public static RemoteFileSystem read(DataInput in) throws IOException { return FileSystemFactory.get(name, type, properties); } - public void write(DataOutput out) throws IOException { - // must write type first - Text.writeString(out, name); + @Override + public void gsonPreProcess() { properties.put(STORAGE_TYPE, type.name()); - out.writeInt(properties.size()); - for (Map.Entry entry : properties.entrySet()) { - Text.writeString(out, entry.getKey()); - Text.writeString(out, entry.getValue()); - } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobFinalOperation.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobFinalOperation.java index 223dbd1ed7bc3af..f536bc8e6c38153 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobFinalOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJobFinalOperation.java @@ -30,7 +30,6 @@ import com.google.gson.reflect.TypeToken; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -56,8 +55,11 @@ public class LoadJobFinalOperation extends TxnCommitAttachment implements Writab @SerializedName(value = "fm") private FailMsg failMsg; // only used for copy into + @SerializedName("cid") private String copyId = ""; + @SerializedName("lfp") private String loadFilePaths = ""; + @SerializedName("prop") private Map properties = new HashMap<>(); public LoadJobFinalOperation() { @@ -125,29 +127,7 @@ public Map getProperties() { return properties; } - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - out.writeLong(id); - loadingStatus.write(out); - out.writeInt(progress); - out.writeLong(loadStartTimestamp); - out.writeLong(finishTimestamp); - Text.writeString(out, jobState.name()); - if (failMsg == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - failMsg.write(out); - } - if (Config.isCloudMode()) { - Text.writeString(out, copyId); - Text.writeString(out, loadFilePaths); - Gson gson = new Gson(); - Text.writeString(out, properties == null ? "" : gson.toJson(properties)); - } - } - + @Deprecated public void readFields(DataInput in) throws IOException { super.readFields(in); id = in.readLong(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MiniLoadTxnCommitAttachment.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MiniLoadTxnCommitAttachment.java index 3bb48f83c6204ec..1f7e7f7b7a7462e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MiniLoadTxnCommitAttachment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MiniLoadTxnCommitAttachment.java @@ -24,7 +24,6 @@ import com.google.gson.annotations.SerializedName; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; public class MiniLoadTxnCommitAttachment extends TxnCommitAttachment { @@ -52,20 +51,7 @@ public String getErrorLogUrl() { return errorLogUrl; } - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - out.writeLong(filteredRows); - out.writeLong(loadedRows); - if (errorLogUrl == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - Text.writeString(out, errorLogUrl); - } - - } - + @Deprecated public void readFields(DataInput in) throws IOException { super.readFields(in); filteredRows = in.readLong(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaProgress.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaProgress.java index 864d5ee0ef7c987..cb6c36dc2e69bd5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaProgress.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaProgress.java @@ -31,11 +31,11 @@ import org.apache.logging.log4j.Logger; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; /** * this is description of kafka routine load progress @@ -56,7 +56,7 @@ public class KafkaProgress extends RoutineLoadProgress { // (partition id, begin offset) // the offset saved here is the next offset need to be consumed @SerializedName(value = "pito") - private Map partitionIdToOffset = Maps.newConcurrentMap(); + private ConcurrentMap partitionIdToOffset = Maps.newConcurrentMap(); public KafkaProgress() { super(LoadDataSourceType.KAFKA); @@ -64,17 +64,24 @@ public KafkaProgress() { public KafkaProgress(TKafkaRLTaskProgress tKafkaRLTaskProgress) { super(LoadDataSourceType.KAFKA); - this.partitionIdToOffset = tKafkaRLTaskProgress.getPartitionCmtOffset(); + this.partitionIdToOffset = new ConcurrentHashMap<>(); + partitionIdToOffset.putAll(tKafkaRLTaskProgress.getPartitionCmtOffset()); } public KafkaProgress(Map partitionIdToOffset) { + super(LoadDataSourceType.KAFKA); + this.partitionIdToOffset = new ConcurrentHashMap<>(); + this.partitionIdToOffset.putAll(partitionIdToOffset); + } + + public KafkaProgress(ConcurrentMap partitionIdToOffset) { super(LoadDataSourceType.KAFKA); this.partitionIdToOffset = partitionIdToOffset; } - public Map getPartitionIdToOffset(List partitionIds) { - Map result = Maps.newHashMap(); - for (Map.Entry entry : partitionIdToOffset.entrySet()) { + public ConcurrentMap getPartitionIdToOffset(List partitionIds) { + ConcurrentMap result = Maps.newConcurrentMap(); + for (ConcurrentMap.Entry entry : partitionIdToOffset.entrySet()) { for (Integer partitionId : partitionIds) { if (entry.getKey().equals(partitionId)) { result.put(partitionId, entry.getValue()); @@ -92,7 +99,7 @@ public Long getOffsetByPartition(int kafkaPartition) { return partitionIdToOffset.get(kafkaPartition); } - public Map getOffsetByPartition() { + public ConcurrentMap getOffsetByPartition() { return partitionIdToOffset; } @@ -109,8 +116,8 @@ public boolean hasPartition() { // OFFSET_END: user set offset = OFFSET_END, no committed msg // OFFSET_BEGINNING: user set offset = OFFSET_BEGINNING, no committed msg // other: current committed msg's offset - private void getReadableProgress(Map showPartitionIdToOffset) { - for (Map.Entry entry : partitionIdToOffset.entrySet()) { + private void getReadableProgress(ConcurrentMap showPartitionIdToOffset) { + for (ConcurrentMap.Entry entry : partitionIdToOffset.entrySet()) { if (entry.getValue() == 0) { showPartitionIdToOffset.put(entry.getKey(), OFFSET_ZERO); } else if (entry.getValue() == -1) { @@ -141,7 +148,7 @@ public void modifyOffset(List> kafkaPartitionOffsets) throws public List> getPartitionOffsetPairs(boolean alreadyConsumed) { List> pairs = Lists.newArrayList(); - for (Map.Entry entry : partitionIdToOffset.entrySet()) { + for (ConcurrentMap.Entry entry : partitionIdToOffset.entrySet()) { if (entry.getValue() == 0) { pairs.add(Pair.of(entry.getKey(), OFFSET_ZERO)); } else if (entry.getValue() == -1) { @@ -169,7 +176,7 @@ public List> getPartitionOffsetPairs(boolean alreadyConsum // so the lag should be (4-2=)2. public Map getLag(Map partitionIdWithLatestOffsets) { Map lagMap = Maps.newHashMap(); - for (Map.Entry entry : partitionIdToOffset.entrySet()) { + for (ConcurrentMap.Entry entry : partitionIdToOffset.entrySet()) { if (partitionIdWithLatestOffsets.containsKey(entry.getKey())) { long lag = partitionIdWithLatestOffsets.get(entry.getKey()) - entry.getValue(); lagMap.put(entry.getKey(), lag); @@ -182,7 +189,7 @@ public Map getLag(Map partitionIdWithLatestOffsets @Override public String toString() { - Map showPartitionIdToOffset = Maps.newHashMap(); + ConcurrentMap showPartitionIdToOffset = Maps.newConcurrentMap(); getReadableProgress(showPartitionIdToOffset); return "KafkaProgress [partitionIdToOffset=" + Joiner.on("|").withKeyValueSeparator("_").join(showPartitionIdToOffset) + "]"; @@ -190,7 +197,7 @@ public String toString() { @Override public String toJsonString() { - Map showPartitionIdToOffset = Maps.newHashMap(); + ConcurrentMap showPartitionIdToOffset = Maps.newConcurrentMap(); getReadableProgress(showPartitionIdToOffset); Gson gson = new Gson(); return gson.toJson(showPartitionIdToOffset); @@ -208,20 +215,11 @@ public void update(RLTaskTxnCommitAttachment attachment) { } } - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - out.writeInt(partitionIdToOffset.size()); - for (Map.Entry entry : partitionIdToOffset.entrySet()) { - out.writeInt((Integer) entry.getKey()); - out.writeLong((Long) entry.getValue()); - } - } - + @Deprecated public void readFields(DataInput in) throws IOException { super.readFields(in); int size = in.readInt(); - partitionIdToOffset = new HashMap<>(); + partitionIdToOffset = new ConcurrentHashMap<>(); for (int i = 0; i < size; i++) { partitionIdToOffset.put(in.readInt(), in.readLong()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java index a1ceebd605c1cb3..751516b559a8c8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java @@ -54,6 +54,7 @@ import com.google.common.collect.Maps; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; @@ -61,7 +62,6 @@ import org.apache.logging.log4j.Logger; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -83,9 +83,12 @@ public class KafkaRoutineLoadJob extends RoutineLoadJob { public static final String KAFKA_FILE_CATALOG = "kafka"; public static final String PROP_GROUP_ID = "group.id"; + @SerializedName("bl") private String brokerList; + @SerializedName("tp") private String topic; // optional, user want to load partitions. + @SerializedName("cskp") private List customKafkaPartitions = Lists.newArrayList(); // current kafka partitions is the actual partition which will be fetched private List currentKafkaPartitions = Lists.newArrayList(); @@ -95,6 +98,7 @@ public class KafkaRoutineLoadJob extends RoutineLoadJob { // We should check it by calling isOffsetForTimes() method before use it. private String kafkaDefaultOffSet = ""; // kafka properties ，property prefix will be mapped to kafka custom parameters, which can be extended in the future + @SerializedName("prop") private Map customProperties = Maps.newHashMap(); private Map convertedCustomProperties = Maps.newHashMap(); @@ -638,24 +642,7 @@ protected Map getCustomProperties() { return ret; } - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - Text.writeString(out, brokerList); - Text.writeString(out, topic); - - out.writeInt(customKafkaPartitions.size()); - for (Integer partitionId : customKafkaPartitions) { - out.writeInt(partitionId); - } - - out.writeInt(customProperties.size()); - for (Map.Entry property : customProperties.entrySet()) { - Text.writeString(out, "property." + property.getKey()); - Text.writeString(out, property.getValue()); - } - } - + @Deprecated public void readFields(DataInput in) throws IOException { super.readFields(in); brokerList = Text.readString(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java index 59ef6c7f1bfd861..af390ce094ad73f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java @@ -26,7 +26,6 @@ import com.google.gson.annotations.SerializedName; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; // {"progress": "", "backendId": "", "taskSignature": "", "numOfErrorData": "", @@ -145,17 +144,7 @@ public String toString() { + ", progress=" + progress.toString() + "]"; } - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - out.writeLong(filteredRows); - out.writeLong(loadedRows); - out.writeLong(unselectedRows); - out.writeLong(receivedBytes); - out.writeLong(taskExecutionTimeMs); - progress.write(out); - } - + @Deprecated public void readFields(DataInput in) throws IOException { super.readFields(in); filteredRows = in.readLong(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 348ac00f9da4376..d870922e2d7e365 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -53,12 +53,13 @@ import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.RoutineLoadOperation; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.planner.StreamLoadPlanner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.SqlModeHelper; -import org.apache.doris.system.SystemInfoService; import org.apache.doris.task.LoadTaskInfo; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileType; @@ -78,6 +79,7 @@ import com.google.common.collect.Maps; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; import lombok.Getter; import lombok.Setter; import org.apache.logging.log4j.LogManager; @@ -102,7 +104,9 @@ * The desireTaskConcurrentNum means that user expect the number of concurrent stream load * The routine load job support different streaming medium such as KAFKA */ -public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback implements Writable, LoadTaskInfo { +public abstract class RoutineLoadJob + extends AbstractTxnStateChangeCallback + implements Writable, LoadTaskInfo, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RoutineLoadJob.class); public static final long DEFAULT_MAX_ERROR_NUM = 0; @@ -159,9 +163,13 @@ public boolean isFinalState() { } } + @SerializedName("id") protected long id; + @SerializedName("n") protected String name; + @SerializedName("dbid") protected long dbId; + @SerializedName("tbid") protected long tableId; // this code is used to verify be task request protected long authCode; @@ -172,23 +180,29 @@ public boolean isFinalState() { protected Expr whereExpr; // optional protected Separator columnSeparator; // optional protected Separator lineDelimiter; + @SerializedName("dtcn") protected int desireTaskConcurrentNum; // optional + @SerializedName("st") protected JobState state = JobState.NEED_SCHEDULE; @Getter + @SerializedName("dsrc") protected LoadDataSourceType dataSourceType; // max number of error data in max batch rows * 10 // maxErrorNum / (maxBatchRows * 10) = max error rate of routine load job // if current error rate is more than max error rate, the job will be paused + @SerializedName("men") protected long maxErrorNum = DEFAULT_MAX_ERROR_NUM; // optional protected double maxFilterRatio = DEFAULT_MAX_FILTER_RATIO; protected long execMemLimit = DEFAULT_EXEC_MEM_LIMIT; protected int sendBatchParallelism = DEFAULT_SEND_BATCH_PARALLELISM; protected boolean loadToSingleTablet = DEFAULT_LOAD_TO_SINGLE_TABLET; // include strict mode + @SerializedName("jp") protected Map jobProperties = Maps.newHashMap(); // sessionVariable's name -> sessionVariable's value // we persist these sessionVariables due to the session is not available when replaying the job. + @SerializedName("sv") protected Map sessionVariables = Maps.newHashMap(); /* @@ -197,8 +211,11 @@ public boolean isFinalState() { * If a task can consume data from source at rate of 10MB/s, and 500B a row, * then we can process 100MB for 10 secs, which is 200000 rows */ + @SerializedName("mbis") protected long maxBatchIntervalS = DEFAULT_MAX_INTERVAL_SECOND; + @SerializedName("mbr") protected long maxBatchRows = DEFAULT_MAX_BATCH_ROWS; + @SerializedName("mbsb") protected long maxBatchSizeBytes = DEFAULT_MAX_BATCH_SIZE; protected boolean isPartialUpdate = false; @@ -222,6 +239,7 @@ public boolean isFinalState() { protected int currentTaskConcurrentNum; + @SerializedName("pg") protected RoutineLoadProgress progress; protected long latestResumeTimestamp; // the latest resume time @@ -231,10 +249,14 @@ public boolean isFinalState() { protected ErrorReason pauseReason; protected ErrorReason cancelReason; + @SerializedName("cts") protected long createTimestamp = System.currentTimeMillis(); + @SerializedName("pts") protected long pauseTimestamp = -1; + @SerializedName("ets") protected long endTimestamp = -1; + @SerializedName("js") protected RoutineLoadStatistic jobStatistic = new RoutineLoadStatistic(); // The tasks belong to this job @@ -245,10 +267,13 @@ public boolean isFinalState() { // this is the origin stmt of CreateRoutineLoadStmt, we use it to persist the RoutineLoadJob, // because we can not serialize the Expressions contained in job. + @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) + @SerializedName("ui") protected UserIdentity userIdentity; + @SerializedName("cm") protected String comment = ""; protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); @@ -261,6 +286,7 @@ public boolean isFinalState() { protected boolean isTypeRead = false; + @SerializedName("ccid") private String cloudClusterId; protected byte enclose = 0; @@ -1799,68 +1825,54 @@ public boolean isFinal() { } public static RoutineLoadJob read(DataInput in) throws IOException { - RoutineLoadJob job = null; - LoadDataSourceType type = LoadDataSourceType.valueOf(Text.readString(in)); - if (type == LoadDataSourceType.KAFKA) { - job = new KafkaRoutineLoadJob(); + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + RoutineLoadJob job = null; + LoadDataSourceType type = LoadDataSourceType.valueOf(Text.readString(in)); + if (type == LoadDataSourceType.KAFKA) { + job = new KafkaRoutineLoadJob(); + } else { + throw new IOException("Unknown load data source type: " + type.name()); + } + + job.setTypeRead(true); + job.readFields(in); + return job; } else { - throw new IOException("Unknown load data source type: " + type.name()); + return GsonUtils.GSON.fromJson(Text.readString(in), RoutineLoadJob.class); } - - job.setTypeRead(true); - job.readFields(in); - return job; } @Override public void write(DataOutput out) throws IOException { - // ATTN: must write type first - Text.writeString(out, dataSourceType.name()); - - out.writeLong(id); - Text.writeString(out, name); - Text.writeString(out, SystemInfoService.DEFAULT_CLUSTER); - out.writeLong(dbId); - out.writeLong(tableId); - out.writeInt(desireTaskConcurrentNum); - Text.writeString(out, state.name()); - out.writeLong(maxErrorNum); - out.writeLong(maxBatchIntervalS); - out.writeLong(maxBatchRows); - out.writeLong(maxBatchSizeBytes); - progress.write(out); - - out.writeLong(createTimestamp); - out.writeLong(pauseTimestamp); - out.writeLong(endTimestamp); - - this.jobStatistic.write(out); - - origStmt.write(out); - out.writeInt(jobProperties.size()); - for (Map.Entry entry : jobProperties.entrySet()) { - Text.writeString(out, entry.getKey()); - Text.writeString(out, entry.getValue()); - } - - out.writeInt(sessionVariables.size()); - for (Map.Entry entry : sessionVariables.entrySet()) { - Text.writeString(out, entry.getKey()); - Text.writeString(out, entry.getValue()); - } - - if (userIdentity == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - userIdentity.write(out); + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + @Override + public void gsonPostProcess() throws IOException { + if (tableId == 0) { + isMultiTable = true; } - if (Config.isCloudMode()) { - Text.writeString(out, cloudClusterId); + jobProperties.forEach((k, v) -> { + if (k.equals(CreateRoutineLoadStmt.PARTIAL_COLUMNS)) { + isPartialUpdate = Boolean.parseBoolean(v); + } + }); + SqlParser parser = new SqlParser(new SqlScanner(new StringReader(origStmt.originStmt), + Long.valueOf(sessionVariables.get(SessionVariable.SQL_MODE)))); + CreateRoutineLoadStmt stmt = null; + try { + stmt = (CreateRoutineLoadStmt) SqlParserUtils.getStmt(parser, origStmt.idx); + stmt.checkLoadProperties(); + setRoutineLoadDesc(stmt.getRoutineLoadDesc()); + } catch (Exception e) { + throw new IOException("error happens when parsing create routine load stmt: " + origStmt.originStmt, e); + } + if (userIdentity != null) { + userIdentity.setIsAnalyzed(); } - Text.writeString(out, comment); } + @Deprecated protected void readFields(DataInput in) throws IOException { if (!isTypeRead) { dataSourceType = LoadDataSourceType.valueOf(Text.readString(in)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadProgress.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadProgress.java index fad465c9277496e..bb3b3e88daa77bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadProgress.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadProgress.java @@ -18,15 +18,13 @@ package org.apache.doris.load.routineload; import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; import com.google.gson.annotations.SerializedName; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; -public abstract class RoutineLoadProgress implements Writable { +public abstract class RoutineLoadProgress { @SerializedName(value = "ldst") protected LoadDataSourceType loadDataSourceType; protected boolean isTypeRead = false; @@ -57,12 +55,7 @@ public static RoutineLoadProgress read(DataInput in) throws IOException { return progress; } - @Override - public void write(DataOutput out) throws IOException { - // ATTN: must write type first - Text.writeString(out, loadDataSourceType.name()); - } - + @Deprecated public void readFields(DataInput in) throws IOException { if (!isTypeRead) { loadDataSourceType = LoadDataSourceType.valueOf(Text.readString(in)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadStatistic.java index 45c9cb1686cba77..ad10367f9823d3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadStatistic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadStatistic.java @@ -18,7 +18,6 @@ package org.apache.doris.load.routineload; import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Maps; @@ -26,12 +25,11 @@ import com.google.gson.annotations.SerializedName; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; import java.util.Map; import java.util.Set; -public class RoutineLoadStatistic implements Writable { +public class RoutineLoadStatistic { /* * The following variables are for statistics * currentErrorRows/currentTotalRows: the row statistics of current sampling period @@ -68,12 +66,6 @@ public class RoutineLoadStatistic implements Writable { // No need to persist, only for tracing txn of routine load job. public Set runningTxnIds = Sets.newHashSet(); - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } - public static RoutineLoadStatistic read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, RoutineLoadStatistic.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyInfo.java index e7ee45174833b47..e1616ea133752e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyInfo.java @@ -17,21 +17,28 @@ package org.apache.doris.mysql.privilege; +import org.apache.doris.catalog.Env; import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.List; -public class UserPropertyInfo implements Writable { +public class UserPropertyInfo implements Writable, GsonPostProcessable { + @SerializedName("u") private String user; + @SerializedName("prop") private List> properties = Lists.newArrayList(); private UserPropertyInfo() { @@ -52,21 +59,26 @@ public List> getProperties() { } public static UserPropertyInfo read(DataInput in) throws IOException { - UserPropertyInfo info = new UserPropertyInfo(); - info.readFields(in); - return info; + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + UserPropertyInfo info = new UserPropertyInfo(); + info.readFields(in); + return info; + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), UserPropertyInfo.class); + } + } + + @Override + public void gsonPostProcess() throws IOException { + user = ClusterNamespace.getNameFromFullName(user); } @Override public void write(DataOutput out) throws IOException { - Text.writeString(out, user); - out.writeInt(properties.size()); - for (Pair entry : properties) { - Text.writeString(out, entry.first); - Text.writeString(out, entry.second); - } + Text.writeString(out, GsonUtils.GSON.toJson(this)); } + @Deprecated public void readFields(DataInput in) throws IOException { user = ClusterNamespace.getNameFromFullName(Text.readString(in)); int size = in.readInt(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AnalyzeDeletionLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AnalyzeDeletionLog.java index 7535aeb29b40682..86dcc506cc846ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AnalyzeDeletionLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AnalyzeDeletionLog.java @@ -17,7 +17,13 @@ package org.apache.doris.persist; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; import java.io.DataInput; import java.io.DataOutput; @@ -25,6 +31,7 @@ public class AnalyzeDeletionLog implements Writable { + @SerializedName("id") public final long id; public AnalyzeDeletionLog(long id) { @@ -33,10 +40,14 @@ public AnalyzeDeletionLog(long id) { @Override public void write(DataOutput out) throws IOException { - out.writeLong(id); + Text.writeString(out, GsonUtils.GSON.toJson(this)); } - public static AnalyzeDeletionLog read(DataInput dataInput) throws IOException { - return new AnalyzeDeletionLog(dataInput.readLong()); + public static AnalyzeDeletionLog read(DataInput in) throws IOException { + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + return new AnalyzeDeletionLog(in.readLong()); + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), AnalyzeDeletionLog.class); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java index 7ed43e2371c0cb4..1f2bcc15eb4252d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java @@ -17,8 +17,10 @@ package org.apache.doris.persist; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Table; import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonPostProcessable; @@ -58,17 +60,22 @@ public Table getTable() { return table; } + @Override public void write(DataOutput out) throws IOException { - Text.writeString(out, dbName); - table.write(out); + Text.writeString(out, GsonUtils.GSON.toJson(this)); } public static CreateTableInfo read(DataInput in) throws IOException { - CreateTableInfo createTableInfo = new CreateTableInfo(); - createTableInfo.readFields(in); - return createTableInfo; + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + CreateTableInfo createTableInfo = new CreateTableInfo(); + createTableInfo.readFields(in); + return createTableInfo; + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), CreateTableInfo.class); + } } + @Deprecated private void readFields(DataInput in) throws IOException { dbName = ClusterNamespace.getNameFromFullName(Text.readString(in)); table = Table.read(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/HbPackage.java b/fe/fe-core/src/main/java/org/apache/doris/persist/HbPackage.java index 17eb7bdc576adfd..5b0c1a8e5c3abdb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/HbPackage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/HbPackage.java @@ -17,10 +17,15 @@ package org.apache.doris.persist; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.system.HeartbeatResponse; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import java.io.DataInput; import java.io.DataOutput; @@ -29,6 +34,7 @@ public class HbPackage implements Writable { + @SerializedName("hbr") private List hbResults = Lists.newArrayList(); public HbPackage() { @@ -44,19 +50,21 @@ public List getHbResults() { } public static HbPackage read(DataInput in) throws IOException { - HbPackage hbPackage = new HbPackage(); - hbPackage.readFields(in); - return hbPackage; + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + HbPackage hbPackage = new HbPackage(); + hbPackage.readFields(in); + return hbPackage; + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), HbPackage.class); + } } @Override public void write(DataOutput out) throws IOException { - out.writeInt(hbResults.size()); - for (HeartbeatResponse heartbeatResult : hbResults) { - heartbeatResult.write(out); - } + Text.writeString(out, GsonUtils.GSON.toJson(this)); } + @Deprecated public void readFields(DataInput in) throws IOException { int size = in.readInt(); for (int i = 0; i < size; i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java index 09d1b61c93667d6..9617a4b8323f887 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java @@ -134,6 +134,7 @@ public static PartitionPersistInfo read(DataInput in) throws IOException { } } + @Deprecated public void readFields(DataInput in) throws IOException { dbId = in.readLong(); tableId = in.readLong(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/ReplicaPersistInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/ReplicaPersistInfo.java index acf9ad5d7367375..1b45bbb9bfc093b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/ReplicaPersistInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/ReplicaPersistInfo.java @@ -17,13 +17,20 @@ package org.apache.doris.persist; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -public class ReplicaPersistInfo implements Writable { +public class ReplicaPersistInfo implements Writable, GsonPostProcessable { public enum ReplicaOperationType { ADD(0), @@ -42,6 +49,7 @@ public enum ReplicaOperationType { DEFAULT_OP(8), TABLET_INFO(9); + @SerializedName("v") private final int value; ReplicaOperationType(int value) { @@ -81,29 +89,46 @@ public static ReplicaOperationType findByValue(int value) { } // required + @SerializedName("op") private ReplicaOperationType opType; + @SerializedName("dbid") private long dbId; + @SerializedName("tbid") private long tableId; + @SerializedName("pid") private long partitionId; + @SerializedName("ind") private long indexId; + @SerializedName("tblid") private long tabletId; + @SerializedName("repid") private long replicaId; + @SerializedName("beid") private long backendId; + @SerializedName("ver") private long version; @Deprecated + @SerializedName("verh") private long versionHash = 0L; + @SerializedName("sh") private int schemaHash = -1; + @SerializedName("ds") private long dataSize; private long remoteDataSize; + @SerializedName("rc") private long rowCount; + @SerializedName("lfv") private long lastFailedVersion = -1L; @Deprecated + @SerializedName("lfvh") private long lastFailedVersionHash = 0L; + @SerializedName("lsv") private long lastSuccessVersion = -1L; @Deprecated + @SerializedName("lsvh") private long lastSuccessVersionHash = 0L; public static ReplicaPersistInfo createForAdd(long dbId, long tableId, long partitionId, long indexId, @@ -283,34 +308,28 @@ public long getLastSuccessVersion() { } public static ReplicaPersistInfo read(DataInput in) throws IOException { - ReplicaPersistInfo replicaInfo = new ReplicaPersistInfo(); - replicaInfo.readFields(in); - return replicaInfo; + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + ReplicaPersistInfo replicaInfo = new ReplicaPersistInfo(); + replicaInfo.readFields(in); + return replicaInfo; + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), ReplicaPersistInfo.class); + } } @Override public void write(DataOutput out) throws IOException { - out.writeLong(dbId); - out.writeLong(tableId); - out.writeLong(partitionId); - out.writeLong(indexId); - out.writeLong(tabletId); - out.writeLong(backendId); - out.writeLong(replicaId); - out.writeLong(version); - out.writeLong(versionHash); - out.writeLong(dataSize); - out.writeLong(rowCount); - - out.writeInt(opType.value); - out.writeLong(lastFailedVersion); - out.writeLong(lastFailedVersionHash); - out.writeLong(lastSuccessVersion); - out.writeLong(lastSuccessVersionHash); - - out.writeInt(schemaHash); + Text.writeString(out, GsonUtils.GSON.toJson(this)); } + @Override + public void gsonPostProcess() throws IOException { + if (opType == null) { + throw new IOException("could not parse operation type from replica info"); + } + } + + @Deprecated public void readFields(DataInput in) throws IOException { dbId = in.readLong(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/RoutineLoadOperation.java b/fe/fe-core/src/main/java/org/apache/doris/persist/RoutineLoadOperation.java index b8daa485da02fba..aa39788f4f02ecb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/RoutineLoadOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/RoutineLoadOperation.java @@ -17,16 +17,23 @@ package org.apache.doris.persist; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.load.routineload.RoutineLoadJob.JobState; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; public class RoutineLoadOperation implements Writable { + @SerializedName("id") private long id; + @SerializedName("js") private JobState jobState; private RoutineLoadOperation() { @@ -46,17 +53,21 @@ public JobState getJobState() { } public static RoutineLoadOperation read(DataInput in) throws IOException { - RoutineLoadOperation operation = new RoutineLoadOperation(); - operation.readFields(in); - return operation; + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + RoutineLoadOperation operation = new RoutineLoadOperation(); + operation.readFields(in); + return operation; + } else { + return GsonUtils.GSON.fromJson(Text.readString(in), RoutineLoadOperation.class); + } } @Override public void write(DataOutput out) throws IOException { - out.writeLong(id); - Text.writeString(out, jobState.name()); + Text.writeString(out, GsonUtils.GSON.toJson(this)); } + @Deprecated public void readFields(DataInput in) throws IOException { id = in.readLong(); jobState = JobState.valueOf(Text.readString(in)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonPreProcessable.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonPreProcessable.java new file mode 100644 index 000000000000000..19a8450e62e3072 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonPreProcessable.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist.gson; + +import java.io.IOException; + +public interface GsonPreProcessable { + public void gsonPreProcess() throws IOException; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 90a1c507011cdd4..7e49fd4e96e1046 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -168,9 +168,9 @@ import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalog; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalTable; +import org.apache.doris.fs.PersistentFileSystem; import org.apache.doris.fs.remote.BrokerFileSystem; import org.apache.doris.fs.remote.ObjFileSystem; -import org.apache.doris.fs.remote.RemoteFileSystem; import org.apache.doris.fs.remote.S3FileSystem; import org.apache.doris.fs.remote.dfs.DFSFileSystem; import org.apache.doris.fs.remote.dfs.JFSFileSystem; @@ -189,7 +189,9 @@ import org.apache.doris.load.loadv2.SparkLoadJob.SparkLoadJobStateUpdateInfo; import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.load.routineload.KafkaProgress; +import org.apache.doris.load.routineload.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment; +import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.routineload.RoutineLoadProgress; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.load.sync.SyncJob; @@ -540,8 +542,13 @@ public class GsonUtils { .registerDefaultSubtype(RoutineLoadProgress.class) .registerSubtype(KafkaProgress.class, KafkaProgress.class.getSimpleName()); - private static RuntimeTypeAdapterFactory remoteFileSystemTypeAdapterFactory - = RuntimeTypeAdapterFactory.of(RemoteFileSystem.class, "clazz") + private static RuntimeTypeAdapterFactory routineLoadJobTypeAdapterFactory + = RuntimeTypeAdapterFactory.of(RoutineLoadJob.class, "clazz") + .registerDefaultSubtype(RoutineLoadJob.class) + .registerSubtype(KafkaRoutineLoadJob.class, KafkaRoutineLoadJob.class.getSimpleName()); + + private static RuntimeTypeAdapterFactory remoteFileSystemTypeAdapterFactory + = RuntimeTypeAdapterFactory.of(PersistentFileSystem.class, "clazz") .registerSubtype(BrokerFileSystem.class, BrokerFileSystem.class.getSimpleName()) .registerSubtype(DFSFileSystem.class, DFSFileSystem.class.getSimpleName()) .registerSubtype(JFSFileSystem.class, JFSFileSystem.class.getSimpleName()) @@ -581,6 +588,7 @@ public class GsonUtils { // .registerTypeHierarchyAdapter(Expr.class, new ExprAdapter()) .registerTypeHierarchyAdapter(Multimap.class, new GuavaMultimapAdapter()) .registerTypeAdapterFactory(new PostProcessTypeAdapterFactory()) + .registerTypeAdapterFactory(new PreProcessTypeAdapterFactory()) .registerTypeAdapterFactory(new ExprAdapterFactory()) .registerTypeAdapterFactory(exprAdapterFactory) .registerTypeAdapterFactory(columnTypeAdapterFactory) @@ -603,6 +611,7 @@ public class GsonUtils { .registerTypeAdapterFactory(constraintTypeAdapterFactory) .registerTypeAdapterFactory(txnCommitAttachmentTypeAdapterFactory) .registerTypeAdapterFactory(routineLoadTypeAdapterFactory) + .registerTypeAdapterFactory(routineLoadJobTypeAdapterFactory) .registerTypeAdapterFactory(remoteFileSystemTypeAdapterFactory) .registerTypeAdapterFactory(jobBackupTypeAdapterFactory) .registerTypeAdapterFactory(loadJobTypeAdapterFactory) @@ -904,6 +913,30 @@ public ImmutableList deserialize(final JsonElement json, final Type type, } } + public static class PreProcessTypeAdapterFactory implements TypeAdapterFactory { + + public PreProcessTypeAdapterFactory() { + } + + @Override + public TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter delegate = gson.getDelegateAdapter(this, type); + + return new TypeAdapter() { + public void write(JsonWriter out, T value) throws IOException { + if (value instanceof GsonPreProcessable) { + ((GsonPreProcessable) value).gsonPreProcess(); + } + delegate.write(out, value); + } + + public T read(JsonReader reader) throws IOException { + return delegate.read(reader); + } + }; + } + } + public static class PostProcessTypeAdapterFactory implements TypeAdapterFactory { public PostProcessTypeAdapterFactory() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java b/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java index 0bbcd8dd3a83abc..5da34ae1c2e759d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java @@ -18,20 +18,18 @@ package org.apache.doris.qe; import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; import com.google.gson.annotations.SerializedName; import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; /* * This class represents an origin statement * in multiple statements. */ -public class OriginStatement implements Writable { +public class OriginStatement { // the origin stmt from client. this may includes more than one statement. // eg: "select 1; select 2; select 3" @SerializedName(value = "originStmt") @@ -50,12 +48,6 @@ public static OriginStatement read(DataInput in) throws IOException { return GsonUtils.GSON.fromJson(json, OriginStatement.class); } - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } - @Override public String toString() { return "OriginStatement{" diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java index 8f641070c40e71b..d48d1702e1b2db5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java @@ -27,6 +27,7 @@ import org.apache.doris.thrift.TFileType; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import java.util.List; @@ -130,7 +131,9 @@ default int getStreamPerNode() { } class ImportColumnDescs { + @SerializedName("des") public List descs = Lists.newArrayList(); + @SerializedName("icdr") public boolean isColumnDescsRewrited = false; public List getFileColNames() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java index c60081a4260e7e1..74396538492a484 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java @@ -78,6 +78,7 @@ public enum LoadJobSourceType { ROUTINE_LOAD_TASK(4), // routine load task use this type BATCH_LOAD_JOB(5); // load job v2 for broker load + @SerializedName("f") private final int flag; private LoadJobSourceType(int flag) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TxnCommitAttachment.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TxnCommitAttachment.java index 0ba52962ec177a5..eebd1ec8a5cbda4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TxnCommitAttachment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TxnCommitAttachment.java @@ -17,11 +17,15 @@ package org.apache.doris.transaction; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.load.loadv2.LoadJobFinalOperation; import org.apache.doris.load.loadv2.MiniLoadTxnCommitAttachment; import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TTxnCommitAttachment; import org.apache.doris.transaction.TransactionState.LoadJobSourceType; @@ -31,7 +35,7 @@ import java.io.DataOutput; import java.io.IOException; -public abstract class TxnCommitAttachment implements Writable { +public abstract class TxnCommitAttachment implements Writable, GsonPostProcessable { @SerializedName(value = "sourceType") protected TransactionState.LoadJobSourceType sourceType; protected boolean isTypeRead = false; @@ -58,32 +62,40 @@ public static TxnCommitAttachment fromThrift(TTxnCommitAttachment txnCommitAttac } public static TxnCommitAttachment read(DataInput in) throws IOException { - TxnCommitAttachment attachment = null; - LoadJobSourceType type = LoadJobSourceType.valueOf(Text.readString(in)); - if (type == LoadJobSourceType.ROUTINE_LOAD_TASK) { - attachment = new RLTaskTxnCommitAttachment(); - } else if (type == LoadJobSourceType.BATCH_LOAD_JOB) { - attachment = new LoadJobFinalOperation(); - } else if (type == LoadJobSourceType.BACKEND_STREAMING) { - attachment = new MiniLoadTxnCommitAttachment(); - } else if (type == LoadJobSourceType.FRONTEND) { - // spark load - attachment = new LoadJobFinalOperation(); + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_137) { + TxnCommitAttachment attachment = null; + LoadJobSourceType type = LoadJobSourceType.valueOf(Text.readString(in)); + if (type == LoadJobSourceType.ROUTINE_LOAD_TASK) { + attachment = new RLTaskTxnCommitAttachment(); + } else if (type == LoadJobSourceType.BATCH_LOAD_JOB) { + attachment = new LoadJobFinalOperation(); + } else if (type == LoadJobSourceType.BACKEND_STREAMING) { + attachment = new MiniLoadTxnCommitAttachment(); + } else if (type == LoadJobSourceType.FRONTEND) { + // spark load + attachment = new LoadJobFinalOperation(); + } else { + throw new IOException("Unknown load job source type: " + type.name()); + } + + attachment.setTypeRead(true); + attachment.readFields(in); + return attachment; } else { - throw new IOException("Unknown load job source type: " + type.name()); + return GsonUtils.GSON.fromJson(Text.readString(in), TxnCommitAttachment.class); } + } - attachment.setTypeRead(true); - attachment.readFields(in); - return attachment; + public void gsonPostProcess() { + setTypeRead(true); } @Override public void write(DataOutput out) throws IOException { - // ATTN: must write type first - Text.writeString(out, sourceType.name()); + Text.writeString(out, GsonUtils.GSON.toJson(this)); } + @Deprecated public void readFields(DataInput in) throws IOException { if (!isTypeRead) { sourceType = LoadJobSourceType.valueOf(Text.readString(in)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadTaskSchedulerTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadTaskSchedulerTest.java index c5bf509464e1129..1548017b66115bb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadTaskSchedulerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadTaskSchedulerTest.java @@ -39,6 +39,7 @@ import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingDeque; public class RoutineLoadTaskSchedulerTest { @@ -61,7 +62,7 @@ public void testRunOneCycle(@Injectable KafkaRoutineLoadJob kafkaRoutineLoadJob1 MetaNotFoundException, AnalysisException, LabelAlreadyUsedException, BeginTransactionException { long beId = 100L; - Map partitionIdToOffset = Maps.newHashMap(); + ConcurrentMap partitionIdToOffset = Maps.newConcurrentMap(); partitionIdToOffset.put(1, 100L); partitionIdToOffset.put(2, 200L); KafkaProgress kafkaProgress = new KafkaProgress(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java index 797507e47dece64..03a29fb453f3cd8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java @@ -77,6 +77,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; public class GlobalTransactionMgrTest { @@ -315,7 +316,7 @@ public void testCommitRoutineLoadTransaction(@Injectable TabletCommitInfo tablet Map idToTransactionState = Maps.newHashMap(); idToTransactionState.put(1L, transactionState); Deencapsulation.setField(routineLoadJob, "maxErrorNum", 10); - Map oldKafkaProgressMap = Maps.newHashMap(); + ConcurrentMap oldKafkaProgressMap = Maps.newConcurrentMap(); oldKafkaProgressMap.put(1, 0L); KafkaProgress oldkafkaProgress = new KafkaProgress(); Deencapsulation.setField(oldkafkaProgress, "partitionIdToOffset", oldKafkaProgressMap); @@ -380,7 +381,7 @@ public void testCommitRoutineLoadTransactionWithErrorMax(@Injectable TabletCommi Map idToTransactionState = Maps.newHashMap(); idToTransactionState.put(1L, transactionState); Deencapsulation.setField(routineLoadJob, "maxErrorNum", 10); - Map oldKafkaProgressMap = Maps.newHashMap(); + ConcurrentMap oldKafkaProgressMap = Maps.newConcurrentMap(); oldKafkaProgressMap.put(1, 0L); KafkaProgress oldkafkaProgress = new KafkaProgress(); Deencapsulation.setField(oldkafkaProgress, "partitionIdToOffset", oldKafkaProgressMap); From 8170df94bfe7e221c0dfb56beb274fb7ea9e25f0 Mon Sep 17 00:00:00 2001 From: hui lai <1353307710@qq.com> Date: Wed, 26 Jun 2024 19:19:40 +0800 Subject: [PATCH 13/31] [fix](load) fix no error url if no partition can be found (#36831) ## Proposed changes before ``` Stream load result: { "TxnId": 2014, "Label": "83ba46bd-280c-4e22-b581-4eb126fd49cf", "Comment": "", "TwoPhaseCommit": "false", "Status": "Fail", "Message": "[DATA_QUALITY_ERROR]Encountered unqualified data, stop processing", "NumberTotalRows": 1, "NumberLoadedRows": 1, "NumberFilteredRows": 0, "NumberUnselectedRows": 0, "LoadBytes": 1669, "LoadTimeMs": 58, "BeginTxnTimeMs": 0, "StreamLoadPutTimeMs": 10, "ReadDataTimeMs": 0, "WriteDataTimeMs": 47, "CommitAndPublishTimeMs": 0 } ``` after ``` Stream load result: { "TxnId": 2014, "Label": "83ba46bd-280c-4e22-b581-4eb126fd49cf", "Comment": "", "TwoPhaseCommit": "false", "Status": "Fail", "Message": "[DATA_QUALITY_ERROR]too many filtered rows", "NumberTotalRows": 1, "NumberLoadedRows": 0, "NumberFilteredRows": 1, "NumberUnselectedRows": 0, "LoadBytes": 1669, "LoadTimeMs": 58, "BeginTxnTimeMs": 0, "StreamLoadPutTimeMs": 10, "ReadDataTimeMs": 0, "WriteDataTimeMs": 47, "CommitAndPublishTimeMs": 0, "ErrorURL": "http://XXXX:8040/api/_load_error_log?file=__shard_4/error_log_insert_stmt_c6461270125a615b-2873833fb48d56a3_c6461270125a615b_2873833fb48d56a3" } ``` --- be/src/vec/sink/vrow_distribution.cpp | 17 ++-- .../load_p0/stream_load/test_error_url_1.csv | 1 + .../test_stream_load_error_url.groovy | 94 +++++++++++++++++++ 3 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 regression-test/data/load_p0/stream_load/test_error_url_1.csv diff --git a/be/src/vec/sink/vrow_distribution.cpp b/be/src/vec/sink/vrow_distribution.cpp index 56dbf74e4475196..d45aa2ea911f2ef 100644 --- a/be/src/vec/sink/vrow_distribution.cpp +++ b/be/src/vec/sink/vrow_distribution.cpp @@ -453,22 +453,23 @@ Status VRowDistribution::generate_rows_distribution( _vpartition->set_transformed_slots(partition_cols_idx); } + Status st = Status::OK(); if (_vpartition->is_auto_detect_overwrite()) { // when overwrite, no auto create partition allowed. - RETURN_IF_ERROR(_generate_rows_distribution_for_auto_overwrite( - block.get(), has_filtered_rows, row_part_tablet_ids)); + st = _generate_rows_distribution_for_auto_overwrite(block.get(), has_filtered_rows, + row_part_tablet_ids); } else if (_vpartition->is_auto_partition() && !_deal_batched) { - RETURN_IF_ERROR(_generate_rows_distribution_for_auto_partition( - block.get(), partition_cols_idx, has_filtered_rows, row_part_tablet_ids, - rows_stat_val)); + st = _generate_rows_distribution_for_auto_partition(block.get(), partition_cols_idx, + has_filtered_rows, row_part_tablet_ids, + rows_stat_val); } else { // not auto partition - RETURN_IF_ERROR(_generate_rows_distribution_for_non_auto_partition( - block.get(), has_filtered_rows, row_part_tablet_ids)); + st = _generate_rows_distribution_for_non_auto_partition(block.get(), has_filtered_rows, + row_part_tablet_ids); } filtered_rows = _block_convertor->num_filtered_rows() + _tablet_finder->num_filtered_rows() - prev_filtered_rows; - return Status::OK(); + return st; } // reuse vars for find_tablets diff --git a/regression-test/data/load_p0/stream_load/test_error_url_1.csv b/regression-test/data/load_p0/stream_load/test_error_url_1.csv new file mode 100644 index 000000000000000..69b869011558899 --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_error_url_1.csv @@ -0,0 +1 @@ +2024-01-09 15:40:46|lhCH2|ctxoxUuJnspAIJAmMuVJKh1B2sLXAwr9XWqcKYuH2ygU71QfcCfB1r8kOBFtllQewP3Hqw3dRq0zv0Bnlwm6Q20yPSQkU4gDmXAbEzxVBCSwU|3roU9YLxSVUVS5vIce1RA9wzNxBEcXGXEAonAGORyVwf6JMYYTlHb58we0|WWrrXKsWzMGAaJfw1veudp5sxSd5FjSzFSqfDWUwVGe96SAikxQhZKt4kpwSiINkJRNkUBf|neq__service_li|kodo-testing-kodoasynq.redis|92xlFBZ1FpQFfnt1ysS71TmVxcekRmCqesO2fBUxV2hWK7LeMbAFoGry6IbqbV2c5hKi9l8NuTG35mXnot9QTvjHKqdWkdy86fMn6|4XBHWHKxZOwh5kVsLz3xLy59W2NbwNDmpWpMQBci6h34uiUknTnseKy2F7dAYr6jjzq5VG7ulv2gwHor6V4HnA3U1XyQNoA9wJ8MFT9wt|Bv0Et5tUNM2JjFz0hS3cCBPX2S9begi8p1lv48M6W0PZQxfB3UoTvavaeyxwxhpfYkOsfqGQTCOMQt7Wwyn3dl2MvFqclt80kNYp7D25ZqREAYskBxWPhOSUaXWDTc|nBj3PlVV|uxoqGdn2SJlIaSEdMlb8hKW2XXgQcL4Tcz9rRqkv9O9rIIqYQ|RrHHtJ5BMmSiBGJsY3cgA4JqoI5l1otuYQoiXE70QQ2X17dOiF3t5umoK8pFEPk|x8zRUfaRJFFkdgBYQyzHrp3bhWIboTv8s2TuLxQ7jbAwduLEI0MFZLgxZHYSge8627ZPgiu96Qwqt9hKK7ZHQpYMEjQTlp2GvZwJiiuxDTYefAbrl6P|R9hEZdbWERAggflyCWNbfqBjNLNWSc0Xyu02Zl8OKpSVPrSIbI|KGHWT4C2GQyOXY2PkWsulr0gvBbfYHVMlKgYU6CEkywO8DQqvzPZkL7axRuVXTq6smU9NiT3KMirozpkTkvPL5MlwBYzsj|9CYIs6GGwpeRov|YNTDSLVHGRrevdOPBewx76od1htIsfcM6c8aYNTLrjoQ1jI9N6M2T5c7zsYQ7vZVobDnD3tqYCpxbjJNrI6LyqkKPxXsOQWTXnmhU2MLygRQrBPyPYkNseTIIve9|I5VKX1noAIeW7EgWM4lpEFvmUy7WgullmYtdNWBBkjhTrz7eetkTm6RYh0eH1lC69xfL2NaIcXbzTkwNPPAug1qnsYrfPQ4xdVvSgjJDropG4SMmT6y134AbnBun|k6wuJs0QDf4YmumR0hOoCDVwf18OpL3q1NoyfLZbt2mhY0YrMH7A8JqdTwvvdjMrYLyMCW1zCK8png4|Q7xOuIxRNVQt6wBntMq6nCauyORBXh7BV0Ac7z6SEReRWYBwyr|iZTSBUU7zukS0z9iPO9IfAJu|GXJRuS5mEUB|OTf42X35rt4T5dtmTO7377i0lphYj2uHb7ZUfJ|1117637773.099065|4614638931763895839|5341939051272067353|3954792818428854896|1967|-6700276424423912861|EPaT0V9uLfy3TrB6St44a9MRtxUuHPbg66JH0mCT4UXEcUAYQv2sXO8VNDP7K \ No newline at end of file diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy index e1e5d82494bbd1a..e13db181e99581d 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy @@ -76,4 +76,98 @@ suite("test_stream_load_error_url", "p0") { } finally { sql """ DROP TABLE IF EXISTS ${tableName} """ } + + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `time` DATETIME(6) NULL, + `__docid` VARCHAR(64) NULL, + `__source` TEXT NULL COMMENT 'hidden', + `message` TEXT NULL, + `__namespace` TEXT NULL COMMENT 'hidden', + `source` TEXT NULL, + `service` TEXT NULL, + `container_host` TEXT NULL, + `endpoint` TEXT NULL, + `env` TEXT NULL, + `http_host` TEXT NULL, + `http_method` TEXT NULL, + `http_route` TEXT NULL, + `http_status_code` TEXT NULL, + `http_url` TEXT NULL, + `operation` TEXT NULL, + `project` TEXT NULL, + `source_type` TEXT NULL, + `status` TEXT NULL, + `span_type` TEXT NULL, + `parent_id` TEXT NULL, + `resource` TEXT NULL, + `span_id` TEXT NULL, + `trace_id` TEXT NULL, + `sample_rate` DOUBLE NULL, + `date` BIGINT NULL, + `create_time` BIGINT NULL, + `priority` BIGINT NULL, + `duration` BIGINT NULL, + `start` BIGINT NULL, + `var` TEXT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`time`, `__docid`) + COMMENT 'default' + PARTITION BY RANGE(`time`) + (PARTITION p20240625 VALUES [('2024-06-25 00:00:00'), ('2024-06-26 00:00:00')), + PARTITION p20240626 VALUES [('2024-06-26 00:00:00'), ('2024-06-27 00:00:00'))) + DISTRIBUTED BY RANDOM BUCKETS AUTO + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "min_load_replica_num" = "-1", + "is_being_synced" = "false", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.time_zone" = "Asia/Shanghai", + "dynamic_partition.start" = "-100000", + "dynamic_partition.end" = "1", + "dynamic_partition.prefix" = "p", + "dynamic_partition.replication_allocation" = "tag.location.default: 1", + "dynamic_partition.buckets" = "10", + "dynamic_partition.create_history_partition" = "false", + "dynamic_partition.history_partition_num" = "16", + "dynamic_partition.hot_partition_num" = "0", + "dynamic_partition.reserved_history_periods" = "NULL", + "dynamic_partition.storage_policy" = "", + "storage_medium" = "hdd", + "storage_format" = "V2", + "inverted_index_storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false", + "group_commit_interval_ms" = "10000", + "group_commit_data_bytes" = "134217728" + ); + """ + + streamLoad { + table "${tableName}" + set 'column_separator', '|' + set 'columns', '`time`,`__docid`,`__source`,`message`,`__namespace`,`source`,`service`,`container_host`,`endpoint`,`env`,`http_host`,`http_method`,`http_route`,`http_status_code`,`http_url`,`operation`,`project`,`source_type`,`status`,`span_type`,`parent_id`,`resource`,`span_id`,`trace_id`,`sample_rate`,`date`,`create_time`,`priority`,`duration`,`start`,`var`' + file 'test_error_url_1.csv' + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("fail", json.Status.toLowerCase()) + assertTrue(json.Message.contains("[DATA_QUALITY_ERROR]too many filtered rows")) + def (code, out, err) = curl("GET", json.ErrorURL) + log.info("error result: " + out) + assertTrue(out.contains("no partition for this tuple")) + log.info("url: " + json.ErrorURL) + } + } + } finally { + sql """ DROP TABLE IF EXISTS ${tableName} """ + } } \ No newline at end of file From cfb878863b38f6a251c04e92c8631836c8393219 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 26 Jun 2024 19:29:11 +0800 Subject: [PATCH 14/31] [improvement](clone) dead be will abort sched task (#36795) When be is down, its related clone task need to abort. Otherwise this task need to wait until timeout. --- .../apache/doris/clone/TabletScheduler.java | 9 ++ .../doris/common/util/DebugPointUtil.java | 10 +- .../org/apache/doris/system/HeartbeatMgr.java | 10 ++ .../doris/clone/BeDownCancelCloneTest.java | 148 ++++++++++++++++++ .../doris/utframe/MockedBackendFactory.java | 8 + 5 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/clone/BeDownCancelCloneTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java index a8093e33e20aa10..55e3ba2e3410b1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -1866,10 +1866,13 @@ private void gatherStatistics(TabletSchedCtx tabletCtx) { * If task is timeout, remove the tablet. */ public void handleRunningTablets() { + Set aliveBeIds = Sets.newHashSet(Env.getCurrentSystemInfo().getAllBackendIds(true)); // 1. remove the tablet ctx if timeout List cancelTablets = Lists.newArrayList(); synchronized (this) { for (TabletSchedCtx tabletCtx : runningTablets.values()) { + long srcBeId = tabletCtx.getSrcBackendId(); + long destBeId = tabletCtx.getDestBackendId(); if (Config.disable_tablet_scheduler) { tabletCtx.setErrMsg("tablet scheduler is disabled"); cancelTablets.add(tabletCtx); @@ -1880,6 +1883,12 @@ public void handleRunningTablets() { tabletCtx.setErrMsg("timeout"); cancelTablets.add(tabletCtx); stat.counterCloneTaskTimeout.incrementAndGet(); + } else if (destBeId > 0 && !aliveBeIds.contains(destBeId)) { + tabletCtx.setErrMsg("dest be " + destBeId + " is dead"); + cancelTablets.add(tabletCtx); + } else if (srcBeId > 0 && !aliveBeIds.contains(srcBeId)) { + tabletCtx.setErrMsg("src be " + srcBeId + " is dead"); + cancelTablets.add(tabletCtx); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugPointUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugPointUtil.java index da06232f0c0f5e4..420cee77631a4e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugPointUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugPointUtil.java @@ -134,12 +134,18 @@ public static void addDebugPoint(String name) { addDebugPoint(name, new DebugPoint()); } - public static void addDebugPointWithValue(String name, E value) { + public static void addDebugPointWithParams(String name, Map params) { DebugPoint debugPoint = new DebugPoint(); - debugPoint.params.put("value", String.format("%s", value)); + debugPoint.params = params; addDebugPoint(name, debugPoint); } + public static void addDebugPointWithValue(String name, E value) { + Map params = Maps.newHashMap(); + params.put("value", String.format("%s", value)); + addDebugPointWithParams(name, params); + } + public static void removeDebugPoint(String name) { DebugPoint debugPoint = debugPoints.remove(name); LOG.info("remove debug point: name={}, exists={}", name, debugPoint != null); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index 5f49e88ba6a0c37..93bc7df083cb3eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -24,6 +24,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.Version; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.persist.HbPackage; import org.apache.doris.resource.Tag; @@ -56,6 +57,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -253,6 +255,14 @@ public HeartbeatResponse call() { result.setBackendInfo(backendInfo); } + String debugDeadBeIds = DebugPointUtil.getDebugParamOrDefault( + "HeartbeatMgr.BackendHeartbeatHandler", "deadBeIds", ""); + if (!Strings.isNullOrEmpty(debugDeadBeIds) + && Arrays.stream(debugDeadBeIds.split(",")).anyMatch(id -> Long.parseLong(id) == backendId)) { + result.getStatus().setStatusCode(TStatusCode.INTERNAL_ERROR); + result.getStatus().addToErrorMsgs("debug point HeartbeatMgr.deadBeIds set dead be"); + } + ok = true; if (result.getStatus().getStatusCode() == TStatusCode.OK) { TBackendInfo tBackendInfo = result.getBackendInfo(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/BeDownCancelCloneTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/BeDownCancelCloneTest.java new file mode 100644 index 000000000000000..06cef5e2446163e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/clone/BeDownCancelCloneTest.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.clone; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.DebugPointUtil; +import org.apache.doris.system.Backend; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class BeDownCancelCloneTest extends TestWithFeService { + + @Override + protected int backendNum() { + return 4; + } + + @Override + protected void beforeCreatingConnectContext() throws Exception { + FeConstants.runningUnitTest = true; + FeConstants.default_scheduler_interval_millisecond = 1000; + Config.enable_debug_points = true; + Config.tablet_checker_interval_ms = 100; + Config.tablet_schedule_interval_ms = 100; + Config.tablet_repair_delay_factor_second = 1; + Config.allow_replica_on_same_host = true; + Config.disable_balance = true; + Config.schedule_batch_size = 1000; + Config.schedule_slot_num_per_hdd_path = 1000; + Config.heartbeat_interval_second = 5; + Config.max_backend_heartbeat_failure_tolerance_count = 1; + Config.min_clone_task_timeout_sec = 20 * 60 * 1000; + } + + @Test + public void test() throws Exception { + connectContext = createDefaultCtx(); + + createDatabase("db1"); + System.out.println(Env.getCurrentInternalCatalog().getDbNames()); + + // 3. create table tbl1 + createTable("create table db1.tbl1(k1 int) distributed by hash(k1) buckets 1;"); + RebalancerTestUtil.updateReplicaPathHash(); + + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("db1"); + OlapTable tbl = (OlapTable) db.getTableOrMetaException("tbl1"); + Assertions.assertNotNull(tbl); + Tablet tablet = tbl.getPartitions().iterator().next() + .getMaterializedIndices(MaterializedIndex.IndexExtState.ALL).iterator().next() + .getTablets().iterator().next(); + + Assertions.assertEquals(3, tablet.getReplicas().size()); + long destBeId = Env.getCurrentSystemInfo().getAllBackendIds(true).stream() + .filter(beId -> tablet.getReplicaByBackendId(beId) == null) + .findFirst() + .orElse(-1L); + Assertions.assertTrue(destBeId != -1L); + Backend destBe = Env.getCurrentSystemInfo().getBackend(destBeId); + Assertions.assertNotNull(destBe); + Assertions.assertTrue(destBe.isAlive()); + + // add debug point, make clone wait + DebugPointUtil.addDebugPoint("MockedBackendFactory.handleCloneTablet.block"); + + // move replica[0] to destBeId + Replica srcReplica = tablet.getReplicas().get(0); + String moveTabletSql = "ADMIN SET REPLICA STATUS PROPERTIES(\"tablet_id\" = \"" + tablet.getId() + "\", " + + "\"backend_id\" = \"" + srcReplica.getBackendId() + "\", \"status\" = \"drop\")"; + Assertions.assertNotNull(getSqlStmtExecutor(moveTabletSql)); + Assertions.assertFalse(srcReplica.isScheduleAvailable()); + + Thread.sleep(3000); + + Assertions.assertEquals(0, Env.getCurrentEnv().getTabletScheduler().getHistoryTablets(100).size()); + Assertions.assertEquals(4, tablet.getReplicas().size()); + Replica destReplica = tablet.getReplicaByBackendId(destBeId); + Assertions.assertNotNull(destReplica); + Assertions.assertEquals(Replica.ReplicaState.CLONE, destReplica.getState()); + + // clone a replica on destBe + List runningTablets = Env.getCurrentEnv().getTabletScheduler().getRunningTablets(100); + Assertions.assertEquals(1, runningTablets.size()); + Assertions.assertEquals(destBeId, runningTablets.get(0).getDestBackendId()); + + Map params2 = Maps.newHashMap(); + params2.put("deadBeIds", String.valueOf(destBeId)); + DebugPointUtil.addDebugPointWithParams("HeartbeatMgr.BackendHeartbeatHandler", params2); + + Thread.sleep((Config.heartbeat_interval_second + * Config.max_backend_heartbeat_failure_tolerance_count + 4) * 1000L); + + destBe = Env.getCurrentSystemInfo().getBackend(destBeId); + Assertions.assertNotNull(destBe); + Assertions.assertFalse(destBe.isAlive()); + + // delete clone dest task + Assertions.assertFalse(Env.getCurrentEnv().getTabletScheduler().getHistoryTablets(100).isEmpty()); + + // first drop dest replica (its backend is dead) and src replica (it's mark as drop) + // then re clone a replica to src be, and waiting for cloning. + runningTablets = Env.getCurrentEnv().getTabletScheduler().getRunningTablets(100); + Assertions.assertEquals(1, runningTablets.size()); + Assertions.assertEquals(srcReplica.getBackendId(), runningTablets.get(0).getDestBackendId()); + + DebugPointUtil.removeDebugPoint("MockedBackendFactory.handleCloneTablet.block"); + Thread.sleep(2000); + + // destBe is dead, cancel clone task + runningTablets = Env.getCurrentEnv().getTabletScheduler().getRunningTablets(100); + Assertions.assertEquals(0, runningTablets.size()); + + Assertions.assertEquals(3, tablet.getReplicas().size()); + for (Replica replica : tablet.getReplicas()) { + Assertions.assertTrue(replica.getBackendId() != destBeId); + Assertions.assertTrue(replica.isScheduleAvailable()); + Assertions.assertEquals(Replica.ReplicaState.NORMAL, replica.getState()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java index 6326c9bc44ab3d6..3934e140f67034f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.DiskInfo; import org.apache.doris.catalog.DiskInfo.DiskState; import org.apache.doris.common.ClientPool; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.proto.Data; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.PBackendServiceGrpc; @@ -247,6 +248,13 @@ private void handleDropTablet(TAgentTaskRequest request, TFinishTaskRequest fini } private void handleCloneTablet(TAgentTaskRequest request, TFinishTaskRequest finishTaskRequest) { + while (DebugPointUtil.isEnable("MockedBackendFactory.handleCloneTablet.block")) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + // ignore + } + } TCloneReq req = request.getCloneReq(); long dataSize = Math.max(1, CatalogTestUtil.getTabletDataSize(req.tablet_id)); long pathHash = req.dest_path_hash; From b2747c5a81171b2fde5530621b0e3c7b7a7bb92a Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 26 Jun 2024 19:36:32 +0800 Subject: [PATCH 15/31] [improvement](balance) partition rebalance chose disk by rr (#36826) Partition rebalance will call function takeAnAvailBalanceSlotFrom to chose disk. But this function prefer to choose the first disk. This PR make this function chose disk by RR. --- .../doris/clone/PartitionRebalancer.java | 5 +- .../apache/doris/clone/TabletScheduler.java | 48 ++++++++------ .../org/apache/doris/clone/PathSlotTest.java | 64 +++++++++++++++++++ 3 files changed, 93 insertions(+), 24 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/clone/PathSlotTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/PartitionRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/clone/PartitionRebalancer.java index 5b5221cad6810e6..23e13e9161b9258 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/PartitionRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/PartitionRebalancer.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.NavigableSet; import java.util.Random; -import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -280,9 +279,9 @@ protected void completeSchedCtx(TabletSchedCtx tabletCtx) Preconditions.checkNotNull(slot, "unable to get slot of toBe " + move.toBe); List paths = beStat.getPathStatistics(); - Set availPath = paths.stream().filter(path -> path.getStorageMedium() == tabletCtx.getStorageMedium() + List availPath = paths.stream().filter(path -> path.getStorageMedium() == tabletCtx.getStorageMedium() && path.isFit(tabletCtx.getTabletSize(), false) == BalanceStatus.OK) - .map(RootPathLoadStatistic::getPathHash).collect(Collectors.toSet()); + .map(RootPathLoadStatistic::getPathHash).collect(Collectors.toList()); long pathHash = slot.takeAnAvailBalanceSlotFrom(availPath); if (pathHash == -1) { throw new SchedException(SchedException.Status.SCHEDULE_FAILED, SubCode.WAITING_SLOT, diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java index 55e3ba2e3410b1c..bf8b2a74f25fda8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -2005,9 +2005,12 @@ public static class PathSlot { // path hash -> slot num private Map pathSlots = Maps.newConcurrentMap(); private long beId; + // only use in takeAnAvailBalanceSlotFrom, make pick RR + private long lastPickPathHash; public PathSlot(Map paths, long beId) { this.beId = beId; + this.lastPickPathHash = -1; for (Map.Entry entry : paths.entrySet()) { pathSlots.put(entry.getKey(), new Slot(entry.getValue())); } @@ -2122,19 +2125,6 @@ public synchronized int getTotalAvailBalanceSlotNum() { return num; } - /** - * get path whose balance slot num is larger than 0 - */ - public synchronized Set getAvailPathsForBalance() { - Set pathHashs = Sets.newHashSet(); - for (Map.Entry entry : pathSlots.entrySet()) { - if (entry.getValue().getAvailableBalance() > 0) { - pathHashs.add(entry.getKey()); - } - } - return pathHashs; - } - public synchronized List> getSlotInfo(long beId) { List> results = Lists.newArrayList(); pathSlots.forEach((key, value) -> { @@ -2167,15 +2157,31 @@ public synchronized long takeBalanceSlot(long pathHash) { return -1; } - public synchronized long takeAnAvailBalanceSlotFrom(Set pathHashs) { - for (Long pathHash : pathHashs) { - Slot slot = pathSlots.get(pathHash); - if (slot == null) { - continue; + public long takeAnAvailBalanceSlotFrom(List pathHashs) { + if (pathHashs.isEmpty()) { + return -1; + } + + Collections.sort(pathHashs); + synchronized (this) { + int preferSlotIndex = pathHashs.indexOf(lastPickPathHash) + 1; + if (preferSlotIndex < 0 || preferSlotIndex >= pathHashs.size()) { + preferSlotIndex = 0; } - if (slot.balanceUsed < slot.getBalanceTotal()) { - slot.balanceUsed++; - return pathHash; + + for (int i = preferSlotIndex; i < pathHashs.size(); i++) { + long pathHash = pathHashs.get(i); + if (takeBalanceSlot(pathHash) != -1) { + lastPickPathHash = pathHash; + return pathHash; + } + } + for (int i = 0; i < preferSlotIndex; i++) { + long pathHash = pathHashs.get(i); + if (takeBalanceSlot(pathHash) != -1) { + lastPickPathHash = pathHash; + return pathHash; + } } } return -1; diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/PathSlotTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/PathSlotTest.java new file mode 100644 index 000000000000000..e26e3042fb8ef9e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/clone/PathSlotTest.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.clone; + +import org.apache.doris.clone.TabletScheduler.PathSlot; +import org.apache.doris.common.Config; +import org.apache.doris.thrift.TStorageMedium; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +class PathSlotTest { + + @Test + public void test() { + Config.balance_slot_num_per_path = 2; + Map paths = Maps.newHashMap(); + List availPathHashs = Lists.newArrayList(); + List expectPathHashs = Lists.newArrayList(); + List gotPathHashs = Lists.newArrayList(); + long startPath = 10001L; + long endPath = 10006L; + for (long pathHash = startPath; pathHash < endPath; pathHash++) { + paths.put(pathHash, TStorageMedium.HDD); + availPathHashs.add(pathHash); + expectPathHashs.add(pathHash); + } + for (long pathHash = startPath; pathHash < endPath; pathHash++) { + expectPathHashs.add(pathHash); + } + for (long pathHash = startPath; pathHash < endPath; pathHash++) { + expectPathHashs.add(-1L); + } + + PathSlot ps = new PathSlot(paths, 1L); + for (int i = 0; i < expectPathHashs.size(); i++) { + Collections.shuffle(availPathHashs); + gotPathHashs.add(ps.takeAnAvailBalanceSlotFrom(availPathHashs)); + } + Assert.assertEquals(expectPathHashs, gotPathHashs); + } + +} From c567ef4ff399af4dc6f8e206d279c99b5bde87c3 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 26 Jun 2024 19:39:55 +0800 Subject: [PATCH 16/31] [chore](rpc) Throw exception when use RPC in ckpt thread or the compatiblility mode (#36856) --- .../org/apache/doris/cloud/rpc/MetaServiceProxy.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index a2dbdaac2c031de..b321b6cffc5b07f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -17,6 +17,7 @@ package org.apache.doris.cloud.rpc; +import org.apache.doris.catalog.Env; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.common.Config; import org.apache.doris.rpc.RpcException; @@ -100,6 +101,16 @@ public void removeProxy(String address) { } private MetaServiceClient getProxy() { + if (Env.isCheckpointThread()) { + LOG.error("You should not use RPC in the checkpoint thread"); + throw new RuntimeException("use RPC in the checkpoint thread"); + } + + if (Config.enable_check_compatibility_mode) { + LOG.error("Should not use RPC in check compatibility mode"); + throw new RuntimeException("use RPC in the check compatibility mode"); + } + String address = Config.meta_service_endpoint; MetaServiceClient service = serviceMap.get(address); if (service != null && service.isNormalState()) { From 4649faf2a841d3d421ee48640770ed1f4e764dbf Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 26 Jun 2024 19:41:42 +0800 Subject: [PATCH 17/31] [fix](load) Fix wrong results for high-concurrent loading (#36841) --- be/src/runtime/group_commit_mgr.cpp | 31 ++++++++++++++++++----------- be/src/runtime/group_commit_mgr.h | 1 + 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp index ab11b795ed51c86..111780c9a421213 100644 --- a/be/src/runtime/group_commit_mgr.cpp +++ b/be/src/runtime/group_commit_mgr.cpp @@ -287,18 +287,25 @@ Status GroupCommitTable::get_first_block_load_queue( if (!_is_creating_plan_fragment) { _is_creating_plan_fragment = true; create_plan_dep->block(); - RETURN_IF_ERROR( - _thread_pool->submit_func([&, be_exe_version, mem_tracker, dep = create_plan_dep] { - Defer defer {[&, dep = dep]() { - dep->set_ready(); - std::unique_lock l(_lock); - _is_creating_plan_fragment = false; - }}; - auto st = _create_group_commit_load(be_exe_version, mem_tracker); - if (!st.ok()) { - LOG(WARNING) << "create group commit load error, st=" << st.to_string(); - } - })); + RETURN_IF_ERROR(_thread_pool->submit_func([&, be_exe_version, mem_tracker, + dep = create_plan_dep] { + Defer defer {[&, dep = dep]() { + dep->set_ready(); + std::unique_lock l(_lock); + for (auto it : _create_plan_deps) { + it->set_ready(); + } + std::vector> {}.swap(_create_plan_deps); + _is_creating_plan_fragment = false; + }}; + auto st = _create_group_commit_load(be_exe_version, mem_tracker); + if (!st.ok()) { + LOG(WARNING) << "create group commit load error, st=" << st.to_string(); + } + })); + } else { + create_plan_dep->block(); + _create_plan_deps.push_back(create_plan_dep); } return try_to_get_matched_queue(); } diff --git a/be/src/runtime/group_commit_mgr.h b/be/src/runtime/group_commit_mgr.h index 702ebb9c7464265..c668197e8dc490d 100644 --- a/be/src/runtime/group_commit_mgr.h +++ b/be/src/runtime/group_commit_mgr.h @@ -187,6 +187,7 @@ class GroupCommitTable { // fragment_instance_id to load_block_queue std::unordered_map> _load_block_queues; bool _is_creating_plan_fragment = false; + std::vector> _create_plan_deps; }; class GroupCommitMgr { From 29eab1ae532352e708832999aadc558a9165c245 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 26 Jun 2024 19:54:45 +0800 Subject: [PATCH 18/31] [feat](Nereids) after partition prune, output rows of scan node only contains rows from selected partitions (#36760) 1. update rowcount if some partitions are pruned 2. refactor StatsCalcualtor for Scan --- .../doris/nereids/StatementContext.java | 12 +- .../nereids/rules/rewrite/ColumnPruning.java | 2 +- .../doris/nereids/stats/StatsCalculator.java | 391 +++++++++++------- .../nereids/trees/plans/algebra/OlapScan.java | 6 + .../LogicalDeferMaterializeOlapScan.java | 5 + .../trees/plans/logical/LogicalOlapScan.java | 8 + .../PhysicalDeferMaterializeOlapScan.java | 5 + .../plans/physical/PhysicalOlapScan.java | 8 + .../doris/statistics/StatisticsBuilder.java | 7 +- .../stats/partition_col_stats.groovy | 2 +- 10 files changed, 297 insertions(+), 149 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 58ccaae34d01073..e79f079129dbd30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids; import org.apache.doris.analysis.StatementBase; -import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Id; @@ -32,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.PlaceholderId; @@ -135,7 +135,7 @@ public class StatementContext implements Closeable { private final Map slotToRelation = Maps.newHashMap(); // the columns in Plan.getExpressions(), such as columns in join condition or filter condition, group by expression - private final Set keyColumns = Sets.newHashSet(); + private final Set keySlots = Sets.newHashSet(); private BitSet disableRules; // table locks @@ -516,12 +516,12 @@ public String toString() { } } - public void addKeyColumn(Column column) { - keyColumns.add(column); + public void addKeySlot(SlotReference slot) { + keySlots.add(slot); } - public boolean isKeyColumn(Column column) { - return keyColumns.contains(column); + public boolean isKeySlot(SlotReference slot) { + return keySlots.contains(slot); } /** Get table id with lazy */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java index e8d4c6d96abbb22..20a91ca56572fea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java @@ -138,7 +138,7 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) { if (stmtContext != null) { for (Slot key : keys) { if (key instanceof SlotReference) { - ((SlotReference) key).getColumn().ifPresent(stmtContext::addKeyColumn); + stmtContext.addKeySlot((SlotReference) key); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index a96ec287f76bd49..bf78fe2a0bf7355 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -18,7 +18,9 @@ package org.apache.doris.nereids.stats; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.FeConstants; @@ -299,15 +301,212 @@ public Statistics visitLogicalFilter(LogicalFilter filter, Void return computeFilter(filter); } + /** + * returns the sum of deltaRowCount for all selected partitions or for the table. + */ + private long computeDeltaRowCount(OlapScan olapScan, SlotReference slot) { + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId()); + + long deltaRowCount = 0; + if (tableMeta != null) { + ColStatsMeta colMeta = tableMeta.findColumnStatsMeta( + olapScan.getTable().getIndexNameById(olapScan.getSelectedIndexId()), slot.getName()); + if (colMeta != null) { + if (olapScan.getSelectedPartitionIds().isEmpty()) { + deltaRowCount = tableMeta.updatedRows.get() - colMeta.updatedRows; + } else { + // sum partition delta row + for (long partitionId : olapScan.getSelectedPartitionIds()) { + deltaRowCount += tableMeta.partitionUpdateRows.getOrDefault(partitionId, 0L) + - colMeta.partitionUpdateRows.getOrDefault(partitionId, 0L); + } + } + } + } + return deltaRowCount; + } + + private void adjustColStats(CatalogRelation catalogRelation, SlotReference slot, + ColumnStatisticBuilder builder) { + if (builder.getAvgSizeByte() <= 0) { + builder.setAvgSizeByte(slot.getDataType().toCatalogDataType().getSlotSize()); + } + if (catalogRelation instanceof OlapScan) { + OlapScan olapScan = (OlapScan) catalogRelation; + long delta = computeDeltaRowCount(olapScan, slot); + if (delta > 0) { + builder.setCount(builder.getCount() + delta); + // clear min-max to avoid error estimation + // for example, after yesterday data loaded, user send query about yesterday immediately. + // since yesterday data are not analyzed, the max date is before yesterday, and hence optimizer + // estimates the filter result is zero + builder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY) + .setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY); + } + } + } + + private ColumnStatistic getColumnStatsFromTableCache(CatalogRelation catalogRelation, SlotReference slot) { + long idxId = -1; + if (catalogRelation instanceof OlapScan) { + idxId = ((OlapScan) catalogRelation).getSelectedIndexIdForMV(); + } + return getColumnStatistic(catalogRelation.getTable(), slot.getName(), idxId); + } + + private ColumnStatistic getColumnStatsFromPartitionCache(CatalogRelation catalogRelation, SlotReference slot, + List partitionNames) { + long idxId = -1; + if (catalogRelation instanceof OlapScan) { + idxId = ((OlapScan) catalogRelation).getSelectedIndexIdForMV(); + } + return getColumnStatistic(catalogRelation.getTable(), slot.getName(), idxId, partitionNames); + } + + private long getSelectedPartitionRowCount(OlapScan olapScan) { + long partRowCountSum = 0; + for (long id : olapScan.getSelectedPartitionIds()) { + long partRowCount = olapScan.getTable().getPartition(id).getBaseIndex().getRowCount(); + // if we cannot get any partition's rowCount, return -1 to fallback to table level stats + if (partRowCount <= 0) { + return -1; + } + partRowCountSum += partRowCount; + } + return partRowCountSum; + } + + private void setHasUnknownColStatsInStatementContext() { + if (ConnectContext.get() != null && ConnectContext.get().getStatementContext() != null) { + ConnectContext.get().getStatementContext().setHasUnknownColStats(true); + } + } + + private void checkIfUnknownStatsUsedAsKey(StatisticsBuilder builder) { + if (ConnectContext.get() != null && ConnectContext.get().getStatementContext() != null) { + for (Map.Entry entry : builder.getExpressionColumnStatsEntries()) { + if (entry.getKey() instanceof SlotReference + && ConnectContext.get().getStatementContext().isKeySlot((SlotReference) entry.getKey())) { + if (entry.getValue().isUnKnown) { + ConnectContext.get().getStatementContext().setHasUnknownColStats(true); + break; + } + } + } + } + } + + private Statistics computeOlapScan(LogicalOlapScan olapScan) { + OlapTable olapTable = olapScan.getTable(); + + if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId() || olapTable instanceof MTMV) { + // mv is selected, return its estimated stats + Optional optStats = cascadesContext.getStatementContext() + .getStatistics(olapScan.getRelationId()); + if (optStats.isPresent()) { + double actualRowCount = olapScan.getTable().getRowCountForNereids(); + // if estimated mv rowCount is more than actual row count, fall back to base table stats + if (actualRowCount > optStats.get().getRowCount()) { + return optStats.get(); + } + } + } + + StatisticsBuilder builder = new StatisticsBuilder(); + + // for system table or FeUt, use ColumnStatistic.UNKNOWN + if (StatisticConstants.isSystemTable(olapTable) || !FeConstants.enableInternalSchemaDb + || ConnectContext.get() == null + || ConnectContext.get().getSessionVariable().internalSession) { + for (Slot slot : olapScan.getOutput()) { + builder.putColumnStatistics(slot, ColumnStatistic.UNKNOWN); + } + setHasUnknownColStatsInStatementContext(); + builder.setRowCount(olapTable.getRowCountForNereids()); + return builder.build(); + } + + // for regression shape test, get row count from columnStats.count + if (ConnectContext.get() == null || !ConnectContext.get().getSessionVariable().enableStats) { + // get row count from any visible slotReference's colStats + double rowCount = 1; + for (Slot slot : olapScan.getOutput()) { + if (isVisibleSlotReference(slot)) { + ColumnStatistic cache = getColumnStatistic(olapTable, slot.getName(), + olapScan.getSelectedIndexIdForMV()); + rowCount = Math.max(rowCount, cache.count); + } + builder.putColumnStatistics(slot, + new ColumnStatisticBuilder(ColumnStatistic.UNKNOWN).setCount(rowCount).build()); + } + setHasUnknownColStatsInStatementContext(); + return builder.setRowCount(rowCount).build(); + } + + // build Stats for olapScan + // if slot is not slotReference or is invisible, use UNKNOWN + List outputSlotReferences = new ArrayList<>(); + for (Slot slot : olapScan.getOutput()) { + if (isVisibleSlotReference(slot)) { + outputSlotReferences.add((SlotReference) slot); + } else { + builder.putColumnStatistics(slot, ColumnStatistic.UNKNOWN); + } + } + // build col stats for outputSlotReferences + if (!olapScan.getSelectedPartitionIds().isEmpty()) { + double rowCount = getSelectedPartitionRowCount(olapScan); + // if partition row count is not available, fallback to table stats + if (rowCount > 0) { + List selectedPartitionNames = new ArrayList<>(olapScan.getSelectedPartitionIds().size()); + olapScan.getSelectedPartitionIds().forEach(id -> { + selectedPartitionNames.add(olapScan.getTable().getPartition(id).getName()); + }); + for (SlotReference slot : outputSlotReferences) { + ColumnStatistic cache = getColumnStatsFromPartitionCache(olapScan, slot, selectedPartitionNames); + ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(cache); + adjustColStats(olapScan, slot, colStatsBuilder); + builder.putColumnStatistics(slot, colStatsBuilder.build()); + rowCount = Math.max(rowCount, colStatsBuilder.getCount()); + } + checkIfUnknownStatsUsedAsKey(builder); + return builder.setRowCount(rowCount).build(); + } + } + + // get table level stats + double rowCount = olapScan.getTable().getRowCountForNereids(); + for (SlotReference slot : outputSlotReferences) { + ColumnStatistic cache = getColumnStatsFromTableCache(olapScan, slot); + ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(cache); + adjustColStats(olapScan, slot, colStatsBuilder); + builder.putColumnStatistics(slot, colStatsBuilder.build()); + rowCount = Math.max(rowCount, colStatsBuilder.getCount()); + } + checkIfUnknownStatsUsedAsKey(builder); + return builder.setRowCount(rowCount).build(); + } + @Override public Statistics visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) { - return computeCatalogRelation(olapScan); + return computeOlapScan(olapScan); + } + + private boolean isVisibleSlotReference(Slot slot) { + if (slot instanceof SlotReference) { + Optional colOpt = ((SlotReference) slot).getColumn(); + if (colOpt.isPresent()) { + return colOpt.get().isVisible(); + } + } + return false; } @Override public Statistics visitLogicalDeferMaterializeOlapScan(LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, Void context) { - return computeCatalogRelation(deferMaterializeOlapScan.getLogicalOlapScan()); + return computeOlapScan(deferMaterializeOlapScan.getLogicalOlapScan()); } @Override @@ -726,6 +925,30 @@ false, getTotalColumnStatisticMap(), false, } } + private ColumnStatistic getColumnStatistic(TableIf table, String colName, long idxId) { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getSessionVariable().internalSession) { + return ColumnStatistic.UNKNOWN; + } + long catalogId; + long dbId; + try { + catalogId = table.getDatabase().getCatalog().getId(); + dbId = table.getDatabase().getId(); + } catch (Exception e) { + // Use -1 for catalog id and db id when failed to get them from metadata. + // This is OK because catalog id and db id is not in the hashcode function of ColumnStatistics cache + // and the table id is globally unique. + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Fail to get catalog id and db id for table %s", table.getName())); + } + catalogId = -1; + dbId = -1; + } + return Env.getCurrentEnv().getStatisticsCache().getColumnStatistics( + catalogId, dbId, table.getId(), idxId, colName); + } + private ColumnStatistic getColumnStatistic(TableIf table, String colName, long idxId, List partitionNames) { ConnectContext connectContext = ConnectContext.get(); if (connectContext != null && connectContext.getSessionVariable().internalSession) { @@ -757,6 +980,7 @@ private ColumnStatistic getColumnStatistic(TableIf table, String colName, long i PartitionColumnStatisticBuilder builder = new PartitionColumnStatisticBuilder(); boolean hasUnknown = false; // check if there is any unknown stats to avoid unnecessary partition column stats merge. + List pColStatsLists = new ArrayList<>(partitionNames.size()); for (String partitionName : partitionNames) { PartitionColumnStatistic pcolStats = Env.getCurrentEnv().getStatisticsCache() .getPartitionColumnStatistics( @@ -764,19 +988,14 @@ private ColumnStatistic getColumnStatistic(TableIf table, String colName, long i if (pcolStats.isUnKnown) { hasUnknown = true; break; + } else { + pColStatsLists.add(pcolStats); } } if (!hasUnknown) { boolean isFirst = true; // try to merge partition column stats - for (String partitionName : partitionNames) { - PartitionColumnStatistic pcolStats = Env.getCurrentEnv().getStatisticsCache() - .getPartitionColumnStatistics( - catalogId, dbId, table.getId(), idxId, partitionName, colName); - if (pcolStats.isUnKnown) { - hasUnknown = true; - break; - } + for (PartitionColumnStatistic pcolStats : pColStatsLists) { if (isFirst) { builder = new PartitionColumnStatisticBuilder(pcolStats); isFirst = false; @@ -784,9 +1003,7 @@ private ColumnStatistic getColumnStatistic(TableIf table, String colName, long i builder.merge(pcolStats); } } - if (!hasUnknown) { - return builder.toColumnStatistics(); - } + return builder.toColumnStatistics(); } } // if any partition-col-stats is unknown, fall back to table level col stats @@ -795,23 +1012,21 @@ private ColumnStatistic getColumnStatistic(TableIf table, String colName, long i } } - // TODO: 1. Subtract the pruned partition - // 2. Consider the influence of runtime filter - // 3. Get NDV and column data size from StatisticManger, StatisticManager doesn't support it now. + /** + * compute stats for catalogRelations except OlapScan + */ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) { - if (catalogRelation instanceof LogicalOlapScan) { - LogicalOlapScan olap = (LogicalOlapScan) catalogRelation; - if (olap.getSelectedIndexId() != olap.getTable().getBaseIndexId()) { - // mv is selected, return its estimated stats - Optional optStats = cascadesContext.getStatementContext() - .getStatistics(olap.getRelationId()); - if (optStats.isPresent()) { - double actualRowCount = catalogRelation.getTable().getRowCountForNereids(); - if (actualRowCount > optStats.get().getRowCount()) { - return optStats.get(); - } - } + StatisticsBuilder builder = new StatisticsBuilder(); + // for FeUt, use ColumnStatistic.UNKNOWN + if (!FeConstants.enableInternalSchemaDb + || ConnectContext.get() == null + || ConnectContext.get().getSessionVariable().internalSession) { + builder.setRowCount(catalogRelation.getTable().getRowCountForNereids()); + for (Slot slot : catalogRelation.getOutput()) { + builder.putColumnStatistics(slot, ColumnStatistic.UNKNOWN); } + setHasUnknownColStatsInStatementContext(); + return builder.build(); } List output = catalogRelation.getOutput(); @@ -822,121 +1037,17 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) { } } Set slotSet = slotSetBuilder.build(); - Map columnStatisticBuilderMap = new HashMap<>(); - TableIf table = catalogRelation.getTable(); - AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); - TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(table.getId()); - long tableUpdatedRows = tableMeta == null ? 0 : tableMeta.updatedRows.get(); - boolean hasUnknownKeyCol = false; - long idxId = -1; - List selectedPartitionNames; - if (catalogRelation instanceof OlapScan) { - OlapScan olapScan = (OlapScan) catalogRelation; - if (olapScan.getTable().getBaseIndexId() != olapScan.getSelectedIndexId()) { - idxId = olapScan.getSelectedIndexId(); - } - selectedPartitionNames = new ArrayList<>(olapScan.getSelectedPartitionIds().size()); - olapScan.getSelectedPartitionIds().forEach(id -> { - selectedPartitionNames.add(olapScan.getTable().getPartition(id).getName()); - }); - } else { - selectedPartitionNames = new ArrayList<>(); - } - double rowCount = 0.0; - for (SlotReference slotReference : slotSet) { - boolean usedAsKey = false; - if (ConnectContext.get() != null && slotReference.getColumn().isPresent() - && ConnectContext.get().getStatementContext() != null) { - usedAsKey = ConnectContext.get().getStatementContext().isKeyColumn(slotReference.getColumn().get()); - } - String colName = slotReference.getColumn().isPresent() - ? slotReference.getColumn().get().getName() - : slotReference.getName(); - boolean shouldIgnoreThisCol = StatisticConstants.shouldIgnoreCol(table, slotReference.getColumn().get()); - if (colName == null) { - throw new RuntimeException(String.format("Invalid slot: %s", slotReference.getExprId())); - } - // compute delta row - long deltaRowCount = 0; - if (catalogRelation instanceof OlapScan) { - OlapTable olapTable = (OlapTable) table; - if (tableMeta != null) { - ColStatsMeta colMeta = tableMeta.findColumnStatsMeta( - olapTable.getIndexNameById(idxId == -1 ? olapTable.getBaseIndexId() : idxId), colName); - if (colMeta != null) { - if (((OlapScan) catalogRelation).getSelectedPartitionIds().isEmpty()) { - deltaRowCount = tableUpdatedRows - colMeta.updatedRows; - } else { - // sum partition delta row - for (long partitionId : ((OlapScan) catalogRelation).getSelectedPartitionIds()) { - deltaRowCount += tableMeta.partitionUpdateRows.getOrDefault(partitionId, 0L) - - colMeta.partitionUpdateRows.getOrDefault(partitionId, 0L); - } - } - } - } - - } - ColumnStatistic cache; - if (!FeConstants.enableInternalSchemaDb - || shouldIgnoreThisCol) { - cache = ColumnStatistic.UNKNOWN; - } else { - cache = getColumnStatistic(table, colName, idxId, selectedPartitionNames); - } + double rowCount = catalogRelation.getTable().getRowCountForNereids(); + for (SlotReference slot : slotSet) { + ColumnStatistic cache = getColumnStatsFromTableCache(catalogRelation, slot); ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(cache); - if (cache.avgSizeByte <= 0) { - colStatsBuilder.setAvgSizeByte(slotReference.getColumn().get().getType().getSlotSize()); - } - if (!cache.isUnKnown) { - rowCount = Math.max(rowCount, cache.count + deltaRowCount); - } else { - if (usedAsKey) { - hasUnknownKeyCol = true; - } - } - if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableStats) { - // deltaRowCount > 0 indicates that - // new data is loaded to the table after this column was analyzed last time. - // In this case, need to eliminate min/max value for this column. - if (deltaRowCount > 0) { - // clear min-max to avoid error estimation - // for example, after yesterday data loaded, user send query about yesterday immediately. - // since yesterday data are not analyzed, the max date is before yesterday, and hence optimizer - // estimates the filter result is zero - colStatsBuilder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY) - .setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY); - if (LOG.isDebugEnabled()) { - LOG.debug("{}.{} is partially analyzed, clear min/max values in column stats", - table.getName(), colName); - } - } - columnStatisticBuilderMap.put(slotReference, colStatsBuilder); - } else { - columnStatisticBuilderMap.put(slotReference, new ColumnStatisticBuilder(ColumnStatistic.UNKNOWN)); - hasUnknownKeyCol = true; - } - } - if (rowCount <= 0.0) { - // if we failed to get rowCount from column stats, then try to get it from TableIf - rowCount = catalogRelation.getTable().getRowCountForNereids(); - } - - if (hasUnknownKeyCol && ConnectContext.get() != null && ConnectContext.get().getStatementContext() != null) { - ConnectContext.get().getStatementContext().setHasUnknownColStats(true); - } - return normalizeCatalogRelationColumnStatsRowCount(rowCount, columnStatisticBuilderMap); - } - - private Statistics normalizeCatalogRelationColumnStatsRowCount(double rowCount, - Map columnStatisticBuilderMap) { - Map columnStatisticMap = new HashMap<>(); - for (Expression slot : columnStatisticBuilderMap.keySet()) { - columnStatisticMap.put(slot, - columnStatisticBuilderMap.get(slot).setCount(rowCount).build()); + adjustColStats(catalogRelation, slot, colStatsBuilder); + rowCount = Math.max(rowCount, colStatsBuilder.getCount()); + builder.putColumnStatistics(slot, colStatsBuilder.build()); } - return new Statistics(rowCount, columnStatisticMap); + checkIfUnknownStatsUsedAsKey(builder); + return builder.build(); } private Statistics computeTopN(TopN topN) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java index d5fe7c2341369d0..65f3639462182a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java @@ -28,6 +28,12 @@ public interface OlapScan { long getSelectedIndexId(); + /** + * if this is mv, return selectedIndexId, o.w -1 + * @return -1 or selectedIndexId + */ + long getSelectedIndexIdForMV(); + List getSelectedPartitionIds(); List getSelectedTabletIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java index 77da6537dd7d095..bff416c19d26105 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java @@ -93,6 +93,11 @@ public long getSelectedIndexId() { return logicalOlapScan.getSelectedIndexId(); } + @Override + public long getSelectedIndexIdForMV() { + return logicalOlapScan.getSelectedIndexIdForMV(); + } + @Override public List getSelectedPartitionIds() { return logicalOlapScan.getSelectedPartitionIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 65e4710836d7e1d..f14cd661d95612e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -333,6 +333,14 @@ public long getSelectedIndexId() { return selectedIndexId; } + @Override + public long getSelectedIndexIdForMV() { + if (getTable().getBaseIndexId() != selectedIndexId) { + return selectedIndexId; + } + return -1; + } + public boolean isIndexSelected() { return indexSelected; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java index f82bd6dbec53420..1acfb8dc4c04b68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java @@ -90,6 +90,11 @@ public long getSelectedIndexId() { return physicalOlapScan.getSelectedIndexId(); } + @Override + public long getSelectedIndexIdForMV() { + return physicalOlapScan.getSelectedIndexIdForMV(); + } + @Override public List getSelectedPartitionIds() { return physicalOlapScan.getSelectedPartitionIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 76713a51e292094..1839f46c552eb3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -90,6 +90,14 @@ public long getSelectedIndexId() { return selectedIndexId; } + @Override + public long getSelectedIndexIdForMV() { + if (getTable().getBaseIndexId() != selectedIndexId) { + return selectedIndexId; + } + return -1; + } + @Override public List getSelectedTabletIds() { return selectedTabletIds; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java index 53d8f49cb14c0f1..29f04f2926ec727 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java @@ -21,11 +21,12 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; public class StatisticsBuilder { private double rowCount; - private int widthInJoinCluster; + private int widthInJoinCluster = 1; private final Map expressionToColumnStats; public StatisticsBuilder() { @@ -60,6 +61,10 @@ public StatisticsBuilder putColumnStatistics(Expression expression, ColumnStatis return this; } + public Set> getExpressionColumnStatsEntries() { + return expressionToColumnStats.entrySet(); + } + public Statistics build() { return new Statistics(rowCount, widthInJoinCluster, expressionToColumnStats); } diff --git a/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy b/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy index 89a32a80d915c8b..3436b6dd86ea110 100644 --- a/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy +++ b/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy @@ -36,7 +36,7 @@ suite("partition_col_stats") { """ //run this sql to make stats be cached sql "select * from pt where k1<3;" - sleep(10) + sleep(10000) explain{ sql "physical plan select * from pt where k1<3;" contains("stats=4") From f71a416d4c63f67e4b4a1e75598c37de5ca259c0 Mon Sep 17 00:00:00 2001 From: hui lai <1353307710@qq.com> Date: Wed, 26 Jun 2024 20:10:08 +0800 Subject: [PATCH 19/31] [chore](query) print query id when killed by timeout checker (#36868) ## Proposed changes ``` 2024-06-26 14:58:30,917 WARN (connect-scheduler-check-timer-0|92) [ConnectContext.checkTimeout():776] kill query timeout, remote: XXXX:XX, query timeout: 900000 ``` It is hard to know which query is killed when timeout, so it is necessary to print query id. --- .../src/main/java/org/apache/doris/qe/ConnectContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 0c276f556861b9d..6b6278db7d69e2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -967,8 +967,8 @@ public void checkTimeout(long now) { // to ms long timeout = getExecTimeout() * 1000L; if (delta > timeout) { - LOG.warn("kill {} timeout, remote: {}, query timeout: {}", - timeoutTag, getRemoteHostPortString(), timeout); + LOG.warn("kill {} timeout, remote: {}, query timeout: {}, query id: {}", + timeoutTag, getRemoteHostPortString(), timeout, queryId); killFlag = true; } } From ec79a94113ae92f81fcfbaf997fd8c12e9bbef5c Mon Sep 17 00:00:00 2001 From: Sun Chenyang Date: Wed, 26 Jun 2024 20:14:22 +0800 Subject: [PATCH 20/31] [fix](regression test) Disable the case in cloud mode (#36769) --- .../inverted_index_p0/test_variant_index_format_v1.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy index 0cd5b9386964b49..627ed987e3a7407 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy +++ b/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy @@ -16,7 +16,9 @@ // under the License. suite("test_variant_index_format_v1", "p0") { - + if (isCloudMode()) { + return; + } def calc_file_crc_on_tablet = { ip, port, tablet -> return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) } From 543576227db1521e66c2d32a6fa522c1a7a7aa61 Mon Sep 17 00:00:00 2001 From: Liu Zhenlong <49094455+Dragonliu2018@users.noreply.github.com> Date: Wed, 26 Jun 2024 20:21:34 +0800 Subject: [PATCH 21/31] [Featrue](default value) add pi as default value (#36280) ## Proposed changes Issue Number: refers https://github.com/apache/doris/issues/34228 add pi as default value, such as: ```sql CREATE TABLE IF NOT EXISTS t1 ( k TINYINT, v double not null DEFAULT PI ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) PROPERTIES("replication_num" = "1"); ``` --- .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 3 +- .../org/apache/doris/nereids/PLParser.g4 | 1 + .../org/apache/doris/analysis/ColumnDef.java | 9 ++ .../nereids/parser/LogicalPlanBuilder.java | 2 + .../plans/commands/info/DefaultValue.java | 5 + .../data/correctness_p0/test_default_pi.out | 28 ++++ .../test_default_pi_streamload.csv | 2 + .../correctness_p0/test_default_pi.groovy | 123 ++++++++++++++++++ 9 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/correctness_p0/test_default_pi.out create mode 100644 regression-test/data/correctness_p0/test_default_pi_streamload.csv create mode 100644 regression-test/suites/correctness_p0/test_default_pi.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 9971d277ebebcc6..5ed655c212a7ce7 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -415,6 +415,7 @@ PERCENT: 'PERCENT'; PERIOD: 'PERIOD'; PERMISSIVE: 'PERMISSIVE'; PHYSICAL: 'PHYSICAL'; +PI: 'PI'; PLACEHOLDER: '?'; PLAN: 'PLAN'; PRIVILEGES: 'PRIVILEGES'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 68da90f9da1ee34..ee9b91fdf6f9c79 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -588,7 +588,7 @@ columnDef ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)? ((NOT)? nullable=NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? - (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | stringValue=STRING_LITERAL + (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | stringValue=STRING_LITERAL | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? (COMMENT comment=STRING_LITERAL)? @@ -1237,6 +1237,7 @@ nonReserved | PERIOD | PERMISSIVE | PHYSICAL + | PI | PLAN | PLUGIN | PLUGINS diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4 index f8dc6039145c0fe..e967da2f95502cd 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4 @@ -884,6 +884,7 @@ non_reserved_words : // Tokens that are not reserved words | PART_LOC | PCTFREE | PCTUSED + | PI | PRECISION | PRESERVE | PRINT diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index f306bb782fe3a66..e01a4f117934557 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -99,6 +99,7 @@ public DefaultValue(boolean isSet, String value, String exprName, Long precision this.defaultValueExprDef = new DefaultValueExprDef(exprName, precision); } + public static String PI = "PI"; public static String CURRENT_DATE = "CURRENT_DATE"; // default "CURRENT_TIMESTAMP", only for DATETIME type public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP"; @@ -526,6 +527,14 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV throw new AnalysisException("Types other than DATE and DATEV2 " + "cannot use current_date as the default value"); } + } else if (null != defaultValueExprDef + && defaultValueExprDef.getExprName().equalsIgnoreCase(DefaultValue.PI)) { + switch (primitiveType) { + case DOUBLE: + break; + default: + throw new AnalysisException("Types other than DOUBLE cannot use pi as the default value"); + } } switch (primitiveType) { case TINYINT: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index f092701fce686e9..d9a0a49753d9b96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2712,6 +2712,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { } } else if (ctx.CURRENT_DATE() != null) { defaultValue = Optional.of(DefaultValue.CURRENT_DATE_DEFAULT_VALUE); + } else if (ctx.PI() != null) { + defaultValue = Optional.of(DefaultValue.PI_DEFAULT_VALUE); } } if (ctx.UPDATE() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java index 48a22daf7b7d6d2..6df1d3d7cf1451f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java @@ -24,6 +24,7 @@ * default value of a column. */ public class DefaultValue { + public static String PI = "PI"; public static String CURRENT_DATE = "CURRENT_DATE"; public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP"; public static String NOW = "now"; @@ -40,6 +41,10 @@ public class DefaultValue { public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO); // default "value", "[]" means empty array public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue("[]"); + // default "value", "3.14159265358979323846" means pi, refer to M_PI in . + // M_PI is enough for type double because the precision of type double is 15~16 digits. + // but it is not adequate for computation using long double. in this case, use M_PIl in . + public static DefaultValue PI_DEFAULT_VALUE = new DefaultValue("3.14159265358979323846", PI); private final String value; // used for column which defaultValue is an expression. diff --git a/regression-test/data/correctness_p0/test_default_pi.out b/regression-test/data/correctness_p0/test_default_pi.out new file mode 100644 index 000000000000000..4b9a36fa12fb5cb --- /dev/null +++ b/regression-test/data/correctness_p0/test_default_pi.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !insert_into1 -- +1 3.141592653589793 1 +2 3.141592653589793 2 +3 3.141592653589793 3 +4 3.141592653589793 4 + +-- !stream_load_csv1 -- +1 3.141592653589793 1 +2 3.141592653589793 2 +3 3.141592653589793 3 +4 3.141592653589793 4 +5 3.141592653589793 5 +6 3.141592653589793 6 + +-- !select_1 -- +1 3.141592653589793 1 +2 3.141592653589793 2 +3 3.141592653589793 3 +4 3.141592653589793 4 + +-- !stream_load_csv1 -- +1 3.141592653589793 1 +2 3.141592653589793 2 +3 3.141592653589793 3 +4 3.141592653589793 4 +5 3.141592653589793 5 +6 3.141592653589793 6 diff --git a/regression-test/data/correctness_p0/test_default_pi_streamload.csv b/regression-test/data/correctness_p0/test_default_pi_streamload.csv new file mode 100644 index 000000000000000..a4f37673aa0b9aa --- /dev/null +++ b/regression-test/data/correctness_p0/test_default_pi_streamload.csv @@ -0,0 +1,2 @@ +5,5 +6,6 diff --git a/regression-test/suites/correctness_p0/test_default_pi.groovy b/regression-test/suites/correctness_p0/test_default_pi.groovy new file mode 100644 index 000000000000000..47333b6a3ad4857 --- /dev/null +++ b/regression-test/suites/correctness_p0/test_default_pi.groovy @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_default_pi") { + def tableName = "test_default_pi" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 DOUBLE NOT NULL DEFAULT PI, + v2 INT + ) + UNIQUE KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + // test insert into. + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + qt_insert_into1 """ select * from ${tableName} order by k; """ + + // test csv stream load. + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'columns', 'k, v1=pi(), v2' + + file 'test_default_pi_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv1 """ select * from ${tableName} order by k; """ + + // test partial update + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 DOUBLE NOT NULL DEFAULT PI, + v2 INT + ) + UNIQUE KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + + qt_select_1 "select * from ${tableName} order by k;" + + streamLoad { + table "${tableName}" + + set 'partial_columns', 'true' + set 'column_separator', ',' + set 'columns', 'k, v1=pi(), v2' + + file 'test_default_pi_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv1 """ select * from ${tableName} order by k; """ + + // test varchar with default pi + sql "DROP TABLE IF EXISTS test_varchar_default" + test { + sql """create table test_varchar_default(a int, b varchar(100) default pi) + distributed by hash(a) properties('replication_num'="1");""" + exception "Types other than DOUBLE cannot use pi as the default value" + } + + // test int with default pi + sql "DROP TABLE IF EXISTS test_int_default" + test { + sql """create table test_int_default(a int, b int default pi) + distributed by hash(a) properties('replication_num'="1");""" + exception "Types other than DOUBLE cannot use pi as the default value" + } + + // test float with default pi + sql "DROP TABLE IF EXISTS test_double_default" + test { + sql """create table test_int_default(a int, b float default pi) + distributed by hash(a) properties('replication_num'="1");""" + exception "Types other than DOUBLE cannot use pi as the default value" + } + +} From 4e1dbb0a9e70ae72e0c8998a83ed18349500099c Mon Sep 17 00:00:00 2001 From: GoGoWen <82132356+GoGoWen@users.noreply.github.com> Date: Wed, 26 Jun 2024 20:28:53 +0800 Subject: [PATCH 22/31] [Enhance](Routine Load) enhance routine load get topic metadata (#35651) ## Proposed changes use _k_consumer->metadata(false /* for this topic */, topic, &metadata, 5000); instead of _k_consumer->metadata(true /* for this topic */, topic, &metadata, 5000); because _k_consumer->metadata(true ...) will get all topic's metadata, which will cause performance issue. Co-authored-by: gogowen2021@gmail.com --- be/src/runtime/routine_load/data_consumer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/runtime/routine_load/data_consumer.cpp b/be/src/runtime/routine_load/data_consumer.cpp index d185a610af74d81..e61e1fdc2e07a79 100644 --- a/be/src/runtime/routine_load/data_consumer.cpp +++ b/be/src/runtime/routine_load/data_consumer.cpp @@ -320,7 +320,7 @@ Status KafkaDataConsumer::get_partition_meta(std::vector* partition_ids // get topic metadata RdKafka::Metadata* metadata = nullptr; RdKafka::ErrorCode err = - _k_consumer->metadata(true /* for this topic */, topic, &metadata, 5000); + _k_consumer->metadata(false /* for this topic */, topic, &metadata, 5000); if (err != RdKafka::ERR_NO_ERROR) { std::stringstream ss; ss << "failed to get partition meta: " << RdKafka::err2str(err); From 1def10f3f58fa10c0d79965589aa8560d8475089 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Wed, 26 Jun 2024 21:30:19 +0800 Subject: [PATCH 23/31] [fix](test)fix regression test case failure (#36391) ## Proposed changes Issue Number: close #xxx --- .../suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy b/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy index 81c712217568796..932ffdbdfb4eb44 100644 --- a/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy +++ b/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy @@ -66,9 +66,9 @@ suite("test_ctas_mv") { sql """ insert into test_table_t1 values(); """ sql """ insert into test_table_t2 values(); """ - sql """ CREATE MATERIALIZED VIEW test_table_view As + createMV(""" CREATE MATERIALIZED VIEW test_table_view As select a1,a3,a4,DATE_FORMAT(a5, 'yyyyMMdd') QUERY_TIME,DATE_FORMAT(a6 ,'yyyyMMdd') CREATE_TIME - from test_table_t1 where DATE_FORMAT(a5, 'yyyyMMdd') =20230131; """ + from test_table_t1 where DATE_FORMAT(a5, 'yyyyMMdd') =20230131; """) sql """ create table szjdf_zjb PROPERTIES("replication_num"="1") as select disf.a2, disf.a1, disf.a3, disf.a4, @@ -90,7 +90,7 @@ suite("test_ctas_mv") { """ sql """ - drop table if exists test_table_t1 FORCE; + drop table if exists test_table_t1; """ sql """ From 2c3a96ed97bef5c9653424fdc61b7b40ff8119a3 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Wed, 26 Jun 2024 22:00:23 +0800 Subject: [PATCH 24/31] [enhancement](compaction) adjust compaction concurrency based on compaction score and workload (#36672) 1 Resolved the issue where the priority queue did not reserve slots for cumulative compaction. 2 When considering compaction task priorities, introduced metrics for CPU and memory usage rates. When the compaction score is low, and CPU or memory usage is high, reduce the number of compaction tasks generated and allocate CPU and memory resources to queries or load. 3 Integrated the logic of the priority queue and concurrency control together, removing the previous priority code. --- be/src/common/config.cpp | 6 ++- be/src/common/config.h | 2 +- be/src/olap/olap_server.cpp | 85 ++++++++++++++++++---------------- be/src/olap/storage_engine.cpp | 20 -------- be/src/util/system_metrics.cpp | 4 ++ be/src/util/system_metrics.h | 2 + 6 files changed, 57 insertions(+), 62 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5eb0e8d26ba4285..4460e477c8f35e3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -437,6 +437,8 @@ DEFINE_Validator(compaction_task_num_per_disk, [](const int config) -> bool { return config >= 2; }); DEFINE_Validator(compaction_task_num_per_fast_disk, [](const int config) -> bool { return config >= 2; }); +DEFINE_Validator(low_priority_compaction_task_num_per_disk, + [](const int config) -> bool { return config >= 2; }); // How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation. DEFINE_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9"); @@ -458,8 +460,8 @@ DEFINE_mInt64(pick_rowset_to_compact_interval_sec, "86400"); // Compaction priority schedule DEFINE_mBool(enable_compaction_priority_scheduling, "true"); -DEFINE_mInt32(low_priority_compaction_task_num_per_disk, "1"); -DEFINE_mDouble(low_priority_tablet_version_num_ratio, "0.7"); +DEFINE_mInt32(low_priority_compaction_task_num_per_disk, "2"); +DEFINE_mInt32(low_priority_compaction_score_threshold, "200"); // Thread count to do tablet meta checkpoint, -1 means use the data directories count. DEFINE_Int32(max_meta_checkpoint_threads, "-1"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 592e96d1dc06ac4..dbf1800270403f6 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -507,7 +507,7 @@ DECLARE_mInt64(pick_rowset_to_compact_interval_sec); // Compaction priority schedule DECLARE_mBool(enable_compaction_priority_scheduling); DECLARE_mInt32(low_priority_compaction_task_num_per_disk); -DECLARE_mDouble(low_priority_tablet_version_num_ratio); +DECLARE_mInt32(low_priority_compaction_score_threshold); // Thread count to do tablet meta checkpoint, -1 means use the data directories count. DECLARE_Int32(max_meta_checkpoint_threads); diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 67d171f1a2280e5..ec4e5843b26b226 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -845,6 +845,42 @@ void StorageEngine::get_tablet_rowset_versions(const PGetTabletVersionsRequest* response->mutable_status()->set_status_code(0); } +bool need_generate_compaction_tasks(int count, int thread_per_disk, CompactionType compaction_type, + bool all_base) { + if (count >= thread_per_disk) { + // Return if no available slot + return false; + } else if (count >= thread_per_disk - 1) { + // Only one slot left, check if it can be assigned to base compaction task. + if (compaction_type == CompactionType::BASE_COMPACTION) { + if (all_base) { + return false; + } + } + } + return true; +} + +int get_concurrent_per_disk(int max_score, int thread_per_disk) { + if (!config::enable_compaction_priority_scheduling) { + return thread_per_disk; + } + + double load_average = DorisMetrics::instance()->system_metrics()->get_load_average_1_min(); + int num_cores = doris::CpuInfo::num_cores(); + bool cpu_usage_high = load_average > num_cores * 0.8; + + auto process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage(); + bool memory_usage_high = process_memory_usage > MemInfo::soft_mem_limit() * 0.8; + + if (max_score <= config::low_priority_compaction_score_threshold && + (cpu_usage_high || memory_usage_high)) { + return config::low_priority_compaction_task_num_per_disk; + } + + return thread_per_disk; +} + std::vector StorageEngine::_generate_compaction_tasks( CompactionType compaction_type, std::vector& data_dirs, bool check_score) { _update_cumulative_compaction_policy(); @@ -874,22 +910,11 @@ std::vector StorageEngine::_generate_compaction_tasks( int count = copied_cumu_map[data_dir].size() + copied_base_map[data_dir].size(); int thread_per_disk = data_dir->is_ssd_disk() ? config::compaction_task_num_per_fast_disk : config::compaction_task_num_per_disk; - if (count >= thread_per_disk) { - // Return if no available slot - need_pick_tablet = false; - if (!check_score) { - continue; - } - } else if (count >= thread_per_disk - 1) { - // Only one slot left, check if it can be assigned to base compaction task. - if (compaction_type == CompactionType::BASE_COMPACTION) { - if (copied_cumu_map[data_dir].empty()) { - need_pick_tablet = false; - if (!check_score) { - continue; - } - } - } + + need_pick_tablet = need_generate_compaction_tasks(count, thread_per_disk, compaction_type, + copied_cumu_map[data_dir].empty()); + if (!need_pick_tablet && !check_score) { + continue; } // Even if need_pick_tablet is false, we still need to call find_best_tablet_to_compaction(), @@ -902,6 +927,9 @@ std::vector StorageEngine::_generate_compaction_tasks( ? copied_cumu_map[data_dir] : copied_base_map[data_dir], &disk_max_score, _cumulative_compaction_policies); + int concurrent_num = get_concurrent_per_disk(disk_max_score, thread_per_disk); + need_pick_tablet = need_generate_compaction_tasks( + count, concurrent_num, compaction_type, copied_cumu_map[data_dir].empty()); for (const auto& tablet : tablets) { if (tablet != nullptr) { if (need_pick_tablet) { @@ -1007,17 +1035,6 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, int64_t permits = 0; Status st = Tablet::prepare_compaction_and_calculate_permits(compaction_type, tablet, compaction, permits); - bool is_low_priority_task = [&]() { - // Can add more strategies to determine whether a task is a low priority task in the future - if (!config::enable_compaction_priority_scheduling) { - return false; - } - if (tablet->version_count() >= - (config::max_tablet_version_num * config::low_priority_tablet_version_num_ratio)) { - return false; - } - return !force; - }(); if (st.ok() && permits > 0) { if (!force) { _permit_limiter.request(permits); @@ -1027,18 +1044,8 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, ? _cumu_compaction_thread_pool : _base_compaction_thread_pool; auto st = thread_pool->submit_func([tablet, compaction = std::move(compaction), - compaction_type, permits, force, is_low_priority_task, - this]() { - if (is_low_priority_task && !_increase_low_priority_task_nums(tablet->data_dir())) { - VLOG_DEBUG << "skip low priority compaction task for tablet: " - << tablet->tablet_id(); - // Todo: push task back - } else { - tablet->execute_compaction(*compaction); - if (is_low_priority_task) { - _decrease_low_priority_task_nums(tablet->data_dir()); - } - } + compaction_type, permits, force, this]() { + tablet->execute_compaction(*compaction); if (!force) { _permit_limiter.release(permits); } diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index cbae6515bb6437d..5d50bb5f4dfdb60 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -1530,26 +1530,6 @@ Status StorageEngine::_persist_broken_paths() { return Status::OK(); } -bool StorageEngine::_increase_low_priority_task_nums(DataDir* dir) { - if (!config::enable_compaction_priority_scheduling) { - return true; - } - std::lock_guard l(_low_priority_task_nums_mutex); - if (_low_priority_task_nums[dir] < config::low_priority_compaction_task_num_per_disk) { - _low_priority_task_nums[dir]++; - return true; - } - return false; -} - -void StorageEngine::_decrease_low_priority_task_nums(DataDir* dir) { - if (config::enable_compaction_priority_scheduling) { - std::lock_guard l(_low_priority_task_nums_mutex); - _low_priority_task_nums[dir]--; - DCHECK(_low_priority_task_nums[dir] >= 0); - } -} - int CreateTabletIdxCache::get_index(const std::string& key) { auto* lru_handle = lookup(key); if (lru_handle) { diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index ee7db9494c2e970..e2fd21024195a48 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -869,6 +869,10 @@ void SystemMetrics::get_disks_io_time(std::map* map) { } } +double SystemMetrics::get_load_average_1_min() { + return _load_average_metrics->load_average_1_minutes->value(); +} + void SystemMetrics::get_network_traffic(std::map* send_map, std::map* rcv_map) { send_map->clear(); diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h index 28db964296d45d7..c72ba3693012fb8 100644 --- a/be/src/util/system_metrics.h +++ b/be/src/util/system_metrics.h @@ -57,6 +57,8 @@ class SystemMetrics { const std::map& lst_rcv_map, int64_t interval_sec, int64_t* send_rate, int64_t* rcv_rate); + double get_load_average_1_min(); + void update_max_disk_io_util_percent(const std::map& lst_value, int64_t interval_sec); void update_max_network_send_bytes_rate(int64_t max_send_bytes_rate); From 8b8be642a92950027bdf172ef04ae94141924244 Mon Sep 17 00:00:00 2001 From: slothever <18522955+wsjz@users.noreply.github.com> Date: Thu, 27 Jun 2024 00:23:42 +0800 Subject: [PATCH 25/31] [regression](kerberos)add hive kerberos docker regression env (#36430) add kerberos docker regression environment add cases for two different hive kerberos --- .../create_kerberos_credential_cache_files.sh | 33 ++++++ .../kerberos/common/conf/doris-krb5.conf | 52 +++++++++ .../common/hadoop/apply-config-overrides.sh | 31 ++++++ .../kerberos/common/hadoop/hadoop-run.sh | 42 ++++++++ .../kerberos/entrypoint-hive-master-2.sh | 36 +++++++ .../kerberos/entrypoint-hive-master.sh | 34 ++++++ .../health-checks/hadoop-health-check.sh | 39 +++++++ .../kerberos/health-checks/health.sh | 34 ++++++ .../docker-compose/kerberos/kerberos.yaml.tpl | 73 +++++++++++++ .../sql/create_kerberos_hive_table.sql | 17 +++ .../two-kerberos-hives/auth-to-local.xml | 29 +++++ .../hive2-default-fs-site.xml | 25 +++++ .../two-kerberos-hives/update-location.sh | 25 +++++ .../thirdparties/run-thirdparties-docker.sh | 30 +++++- .../security/authentication/HadoopUGI.java | 4 +- .../doris/datasource/ExternalCatalog.java | 9 +- .../datasource/hive/HMSExternalCatalog.java | 6 +- .../datasource/hive/HiveMetaStoreCache.java | 4 +- .../hive/HiveMetaStoreClientHelper.java | 4 +- .../iceberg/IcebergMetadataCache.java | 5 +- .../paimon/PaimonExternalCatalog.java | 3 +- .../doris/fs/remote/RemoteFileSystem.java | 5 + .../apache/doris/fs/remote/S3FileSystem.java | 3 +- .../doris/fs/remote/dfs/DFSFileSystem.java | 13 ++- regression-test/conf/regression-conf.groovy | 4 + .../kerberos/test_single_hive_kerberos.out | 6 ++ .../kerberos/test_two_hive_kerberos.out | 12 +++ .../pipeline/external/conf/be.conf | 3 + .../pipeline/external/conf/fe.conf | 2 + .../external/conf/regression-conf.groovy | 5 + .../kerberos/test_single_hive_kerberos.groovy | 101 ++++++++++++++++++ .../kerberos/test_two_hive_kerberos.groovy | 72 +++++++++++++ 32 files changed, 742 insertions(+), 19 deletions(-) create mode 100644 docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh create mode 100644 docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf create mode 100755 docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh create mode 100755 docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh create mode 100755 docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh create mode 100755 docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh create mode 100755 docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh create mode 100644 docker/thirdparties/docker-compose/kerberos/health-checks/health.sh create mode 100644 docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl create mode 100644 docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql create mode 100755 docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml create mode 100755 docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml create mode 100755 docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh create mode 100644 regression-test/data/external_table_p0/kerberos/test_single_hive_kerberos.out create mode 100644 regression-test/data/external_table_p0/kerberos/test_two_hive_kerberos.out create mode 100644 regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy create mode 100644 regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy diff --git a/docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh b/docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh new file mode 100644 index 000000000000000..2bba3f928b198c7 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -exuo pipefail + +TICKET_LIFETIME='30m' + +kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/presto-server-krbcc \ + -kt /etc/trino/conf/presto-server.keytab presto-server/$(hostname -f)@LABS.TERADATA.COM + +kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hive-presto-master-krbcc \ + -kt /etc/trino/conf/hive-presto-master.keytab hive/$(hostname -f)@LABS.TERADATA.COM + +kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hdfs-krbcc \ + -kt /etc/hadoop/conf/hdfs.keytab hdfs/hadoop-master@LABS.TERADATA.COM + +kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hive-krbcc \ + -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM diff --git a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf new file mode 100644 index 000000000000000..7624b94e6ad2a4a --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[logging] + default = FILE:/var/log/krb5libs.log + kdc = FILE:/var/log/krb5kdc.log + admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + default_realm = LABS.TERADATA.COM + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + # this setting is causing a Message stream modified (41) error when talking to KDC running on CentOS 7: https://stackoverflow.com/a/60978520 + # renew_lifetime = 7d + forwardable = true + udp_preference_limit = 1 + +[realms] + LABS.TERADATA.COM = { + kdc = hadoop-master:88 + admin_server = hadoop-master + } + OTHERLABS.TERADATA.COM = { + kdc = hadoop-master:89 + admin_server = hadoop-master + } + OTHERLABS.TERADATA.COM = { + kdc = hadoop-master:89 + admin_server = hadoop-master + } +OTHERREALM.COM = { + kdc = hadoop-master-2:88 + admin_server = hadoop-master + } + +[domain_realm] + hadoop-master-2 = OTHERREALM.COM diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh new file mode 100755 index 000000000000000..ec2dc074e705cdc --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# test whether OVERRIDES_DIR is set +if [[ -n "${OVERRIDES_DIR+x}" ]]; then + echo "The OVERRIDES_DIR (${OVERRIDES_DIR}) support is disabled as it was deemed unused." >&2 + echo "It is being removed." >&2 + exit 16 +fi + +if test -e /overrides; then + find /overrides >&2 + echo "The /overrides handling is disabled as it was deemed unused." >&2 + echo "It is being removed." >&2 + exit 17 +fi diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh new file mode 100755 index 000000000000000..b8bfd8715e95656 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if test $# -gt 0; then + echo "$0 does not accept arguments" >&2 + exit 32 +fi + +set -x + +HADOOP_INIT_D=${HADOOP_INIT_D:-/etc/hadoop-init.d/} + +echo "Applying hadoop init.d scripts from ${HADOOP_INIT_D}" +if test -d "${HADOOP_INIT_D}"; then + for init_script in "${HADOOP_INIT_D}"*; do + chmod a+x "${init_script}" + "${init_script}" + done +fi + +trap exit INT + +echo "Running services with supervisord" + +supervisord -c /etc/supervisord.conf diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh new file mode 100755 index 000000000000000..c21460c3a57a0f5 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +echo "Copying kerberos keytabs to /keytabs/" +mkdir -p /etc/hadoop-init.d/ +cp /etc/trino/conf/hive-presto-master.keytab /keytabs/other-hive-presto-master.keytab +cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab +cp /keytabs/update-location.sh /etc/hadoop-init.d/update-location.sh +/usr/local/hadoop-run.sh & + +sleep 30 + +echo "Init kerberos test data" +kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM +hive -f /usr/local/sql/create_kerberos_hive_table.sql + +sleep 20 + +tail -f /dev/null diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh new file mode 100755 index 000000000000000..62924992219a1d6 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +echo "Copying kerberos keytabs to keytabs/" +mkdir -p /etc/hadoop-init.d/ +cp /etc/trino/conf/* /keytabs/ +/usr/local/hadoop-run.sh & + +sleep 30 + +echo "Init kerberos test data" +kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM +hive -f /usr/local/sql/create_kerberos_hive_table.sql + +sleep 20 + +tail -f /dev/null diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh new file mode 100755 index 000000000000000..190fa838d6f64d1 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if test $# -gt 0; then + echo "$0 does not accept arguments" >&2 + exit 32 +fi + +# Supervisord is not running +if ! test -f /tmp/supervisor.sock; then + exit 0 +fi + +# Check if all Hadoop services are running +FAILED=$(supervisorctl status | grep -v RUNNING || true) + +if [ "$FAILED" == "" ]; then + exit 0 +else + echo "Some of the services are failing: ${FAILED}" + exit 1 +fi diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh new file mode 100644 index 000000000000000..515f37e36ac9e37 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +if test $# -gt 0; then + echo "$0 does not accept arguments" >&2 + exit 32 +fi + +set -x + +HEALTH_D=${HEALTH_D:-/etc/health.d/} + +if test -d "${HEALTH_D}"; then + for health_script in "${HEALTH_D}"/*; do + "${health_script}" &>> /var/log/container-health.log || exit 1 + done +fi diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl new file mode 100644 index 000000000000000..6f175ab9c6ea811 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "3" +services: + hive-krb: + image: ghcr.io/trinodb/testing/hdp3.1-hive-kerberized + container_name: doris--kerberos1 + volumes: + - ./two-kerberos-hives:/keytabs + - ./sql:/usr/local/sql + - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh + - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh + - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh + - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh + hostname: hadoop-master + entrypoint: /usr/local/entrypoint-hive-master.sh + healthcheck: + test: ./health-checks/health.sh + ports: + - "5806:5006" + - "8820:8020" + - "8842:8042" + - "9800:9000" + - "9883:9083" + - "18000:10000" + networks: + doris--krb_net: + ipv4_address: 172.31.71.25 + + hive-krb2: + image: ghcr.io/trinodb/testing/hdp3.1-hive-kerberized-2:96 + container_name: doris--kerberos2 + hostname: hadoop-master-2 + volumes: + - ./two-kerberos-hives:/keytabs + - ./sql:/usr/local/sql + - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh + - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh + - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh + - ./entrypoint-hive-master-2.sh:/usr/local/entrypoint-hive-master-2.sh + entrypoint: /usr/local/entrypoint-hive-master-2.sh + healthcheck: + test: ./health-checks/health.sh + ports: + - "15806:5006" + - "18820:8020" + - "18842:8042" + - "19800:9000" + - "19883:9083" + - "18800:10000" + networks: + doris--krb_net: + ipv4_address: 172.31.71.26 + +networks: + doris--krb_net: + ipam: + config: + - subnet: 172.31.71.0/24 diff --git a/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql b/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql new file mode 100644 index 000000000000000..ecf58e88158b120 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql @@ -0,0 +1,17 @@ +CREATE DATABASE IF NOT EXISTS `test_krb_hive_db`; +CREATE TABLE IF NOT EXISTS `test_krb_hive_db`.`test_krb_hive_tbl`( + `id_key` int, + `string_key` string, + `rate_val` double, + `comment` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; + +INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(1, 'a', 3.16, 'cc0'); +INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(2, 'b', 41.2, 'cc1'); +INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(3, 'c', 6.2, 'cc2'); +INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(4, 'd', 1.4, 'cc3'); diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml new file mode 100755 index 000000000000000..c0ce38e3cdc1bdf --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml @@ -0,0 +1,29 @@ + + + + + hadoop.security.auth_to_local + + RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*// + RULE:[2:$1@$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + DEFAULT + + + diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml new file mode 100755 index 000000000000000..4541c1328aea16e --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml @@ -0,0 +1,25 @@ + + + + + fs.default.name + hdfs://hadoop-master-2:9000 + + diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh new file mode 100755 index 000000000000000..8d727b2308dd05c --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +/usr/bin/mysqld_safe & +while ! mysqladmin ping -proot --silent; do sleep 1; done + +hive --service metatool -updateLocation hdfs://hadoop-master-2:9000/user/hive/warehouse hdfs://hadoop-master:9000/user/hive/warehouse + +killall mysqld +while pgrep mysqld; do sleep 1; done diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index b9c720c457effe8..805c137958ddf1a 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -37,7 +37,7 @@ Usage: $0 --stop stop the specified components All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,lakesoul + mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,lakesoul,kerberos " exit 1 } @@ -59,7 +59,7 @@ eval set -- "${OPTS}" if [[ "$#" == 1 ]]; then # default - COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2" + COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2,kerberos" else while true; do case "$1" in @@ -91,7 +91,7 @@ else done if [[ "${COMPONENTS}"x == ""x ]]; then if [[ "${STOP}" -eq 1 ]]; then - COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,lakesoul" + COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,kerberos,lakesoul" fi fi fi @@ -136,6 +136,7 @@ RUN_SPARK=0 RUN_MARIADB=0 RUN_DB2=0 RUN_LAKESOUL=0 +RUN_KERBEROS=0 for element in "${COMPONENTS_ARR[@]}"; do if [[ "${element}"x == "mysql"x ]]; then @@ -170,6 +171,8 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_DB2=1 elif [[ "${element}"x == "lakesoul"x ]]; then RUN_LAKESOUL=1 + elif [[ "${element}"x == "kerberos"x ]]; then + RUN_KERBEROS=1 else echo "Invalid component: ${element}" usage @@ -545,5 +548,26 @@ if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then # git checkout doris_dev cd LakeSoul/rust cargo test load_tpch_data --package lakesoul-datafusion --features=ci -- --nocapture +fi +if [[ "${RUN_KERBEROS}" -eq 1 ]]; then + echo "RUN_KERBEROS" + cp "${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl "${ROOT}"/docker-compose/kerberos/kerberos.yaml + sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/kerberos/kerberos.yaml + sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml down + sudo rm -rf "${ROOT}"/docker-compose/kerberos/data + if [[ "${STOP}" -ne 1 ]]; then + echo "PREPARE KERBEROS DATA" + rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.keytab + rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.jks + rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.conf + sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml up -d + sudo rm -f /keytabs + sudo ln -s "${ROOT}"/docker-compose/kerberos/two-kerberos-hives /keytabs + sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /keytabs/krb5.conf + sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /etc/krb5.conf + + echo '172.31.71.25 hadoop-master' >> /etc/hosts + echo '172.31.71.26 hadoop-master-2' >> /etc/hosts + fi fi diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java index 342f86b71259e68..5e61664c8fa015d 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java @@ -112,7 +112,9 @@ public static T ugiDoAs(AuthenticationConfig authConf, PrivilegedExceptionAc UserGroupInformation ugi = HadoopUGI.loginWithUGI(authConf); try { if (ugi != null) { - ugi.checkTGTAndReloginFromKeytab(); + if (authConf instanceof KerberosAuthenticationConfig) { + ugi.checkTGTAndReloginFromKeytab(); + } return ugi.doAs(action); } else { return action.run(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 5bdbe594059cb94..929378ae80f48f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -51,6 +51,7 @@ import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; @@ -67,7 +68,6 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -148,7 +148,7 @@ public ExternalCatalog(long catalogId, String name, InitCatalogLog.Type logType, } public Configuration getConfiguration() { - Configuration conf = new HdfsConfiguration(); + Configuration conf = DFSFileSystem.getHdfsConf(ifNotSetFallbackToSimpleAuth()); Map catalogProperties = catalogProperty.getHadoopProperties(); for (Map.Entry entry : catalogProperties.entrySet()) { conf.set(entry.getKey(), entry.getValue()); @@ -181,6 +181,11 @@ public void setDefaultPropsIfMissing(boolean isReplay) { Boolean.valueOf(catalogProperty.getOrDefault(USE_META_CACHE, String.valueOf(DEFAULT_USE_META_CACHE)))); } + // we need check auth fallback for kerberos or simple + public boolean ifNotSetFallbackToSimpleAuth() { + return catalogProperty.getOrDefault(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "").isEmpty(); + } + // Will be called when creating catalog(not replaying). // Subclass can override this method to do some check when creating catalog. public void checkWhenCreating() throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 243dfb3c24f15b7..91192b63c24d6e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -37,6 +37,7 @@ import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.fs.FileSystemProvider; import org.apache.doris.fs.FileSystemProviderImpl; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import org.apache.doris.transaction.TransactionManagerFactory; import com.google.common.base.Strings; @@ -59,7 +60,6 @@ public class HMSExternalCatalog extends ExternalCatalog { public static final String FILE_META_CACHE_TTL_SECOND = "file.meta.cache.ttl-second"; // broker name for file split and query scan. public static final String BIND_BROKER_NAME = "broker.name"; - private static final String PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH = "ipc.client.fallback-to-simple-auth-allowed"; // -1 means file cache no ttl set public static final int FILE_META_CACHE_NO_TTL = -1; @@ -244,9 +244,9 @@ public void notifyPropertiesUpdated(Map updatedProps) { @Override public void setDefaultPropsIfMissing(boolean isReplay) { super.setDefaultPropsIfMissing(isReplay); - if (catalogProperty.getOrDefault(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "").isEmpty()) { + if (ifNotSetFallbackToSimpleAuth()) { // always allow fallback to simple auth, so to support both kerberos and simple auth - catalogProperty.addProperty(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true"); + catalogProperty.addProperty(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index f402d27cf6d8a5e..5e3366e37c4a5d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -41,6 +41,7 @@ import org.apache.doris.fs.FileSystemCache; import org.apache.doris.fs.remote.RemoteFile; import org.apache.doris.fs.remote.RemoteFileSystem; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import org.apache.doris.metric.GaugeMetric; import org.apache.doris.metric.Metric; import org.apache.doris.metric.MetricLabel; @@ -66,7 +67,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -433,7 +433,7 @@ private FileCacheValue loadFiles(FileCacheKey key) { } private synchronized void setJobConf() { - Configuration configuration = new HdfsConfiguration(); + Configuration configuration = DFSFileSystem.getHdfsConf(catalog.ifNotSetFallbackToSimpleAuth()); for (Map.Entry entry : catalog.getCatalogProperty().getHadoopProperties().entrySet()) { configuration.set(entry.getKey(), entry.getValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java index 2e7693619b847c9..ea821cefec6f237 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java @@ -42,13 +42,13 @@ import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.security.authentication.HadoopUGI; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import org.apache.doris.thrift.TExprOpcode; import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.avro.Schema; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -841,7 +841,7 @@ public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) { } public static Configuration getConfiguration(HMSExternalTable table) { - Configuration conf = new HdfsConfiguration(); + Configuration conf = DFSFileSystem.getHdfsConf(table.getCatalog().ifNotSetFallbackToSimpleAuth()); for (Map.Entry entry : table.getHadoopProperties().entrySet()) { conf.set(entry.getKey(), entry.getValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java index 68064c4e439b670..dc11a6cacc24cf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java @@ -25,13 +25,13 @@ import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import org.apache.doris.thrift.TIcebergMetadataParams; import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.SerializableTable; import org.apache.iceberg.Snapshot; @@ -177,7 +177,8 @@ public void invalidateDbCache(long catalogId, String dbName) { private Catalog createIcebergHiveCatalog(String uri, Map hdfsConf, Map props) { // set hdfs configure - Configuration conf = new HdfsConfiguration(); + Configuration conf = DFSFileSystem.getHdfsConf( + hdfsConf.getOrDefault(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "").isEmpty()); for (Map.Entry entry : hdfsConf.entrySet()) { conf.set(entry.getKey(), entry.getValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 116912246815492..8f187e6d7ca800e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -25,6 +25,7 @@ import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.datasource.property.constants.PaimonProperties; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -60,7 +61,7 @@ public PaimonExternalCatalog(long catalogId, String name, String comment) { @Override protected void initLocalObjectsImpl() { - Configuration conf = new Configuration(); + Configuration conf = DFSFileSystem.getHdfsConf(ifNotSetFallbackToSimpleAuth()); for (Map.Entry propEntry : this.catalogProperty.getHadoopProperties().entrySet()) { conf.set(propEntry.getKey(), propEntry.getValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java index 311532794f1f1f2..68de3a8fdef86fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java @@ -21,6 +21,7 @@ import org.apache.doris.backup.Status; import org.apache.doris.common.UserException; import org.apache.doris.fs.PersistentFileSystem; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.fs.FileStatus; @@ -46,6 +47,10 @@ protected org.apache.hadoop.fs.FileSystem nativeFileSystem(String remotePath) th throw new UserException("Not support to getFileSystem."); } + public boolean ifNotSetFallbackToSimpleAuth() { + return properties.getOrDefault(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "").isEmpty(); + } + @Override public Status listFiles(String remotePath, boolean recursive, List result) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java index 3130a0cea522a37..525d80d6797b353 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java @@ -22,6 +22,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.fs.obj.S3ObjStorage; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.google.common.annotations.VisibleForTesting; @@ -60,7 +61,7 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { if (dfsFileSystem == null) { synchronized (this) { if (dfsFileSystem == null) { - Configuration conf = new Configuration(); + Configuration conf = DFSFileSystem.getHdfsConf(ifNotSetFallbackToSimpleAuth()); System.setProperty("com.amazonaws.services.s3.enableV4", "true"); // the entry value in properties may be null, and PropertyConverter.convertToHadoopFSProperties(properties).entrySet().stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java index d608653024fc0a1..5532f1187fe2bd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java @@ -56,8 +56,8 @@ public class DFSFileSystem extends RemoteFileSystem { + public static final String PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH = "ipc.client.fallback-to-simple-auth-allowed"; private static final Logger LOG = LogManager.getLogger(DFSFileSystem.class); - private HDFSFileOperations operations = null; public DFSFileSystem(Map properties) { @@ -75,7 +75,7 @@ public FileSystem nativeFileSystem(String remotePath) throws UserException { if (dfsFileSystem == null) { synchronized (this) { if (dfsFileSystem == null) { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getHdfsConf(ifNotSetFallbackToSimpleAuth()); for (Map.Entry propEntry : properties.entrySet()) { conf.set(propEntry.getKey(), propEntry.getValue()); } @@ -94,6 +94,15 @@ public FileSystem nativeFileSystem(String remotePath) throws UserException { return dfsFileSystem; } + public static Configuration getHdfsConf(boolean fallbackToSimpleAuth) { + Configuration hdfsConf = new HdfsConfiguration(); + if (fallbackToSimpleAuth) { + // need support fallback to simple if the cluster is a mixture of kerberos and simple auth. + hdfsConf.set(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true"); + } + return hdfsConf; + } + @Override public Status downloadWithFileSize(String remoteFilePath, String localFilePath, long fileSize) { if (LOG.isDebugEnabled()) { diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 6e9a162df003c7f..6d4d915633914bf 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -220,3 +220,7 @@ externalEnvIp="127.0.0.1" // trino-connector catalog test config enableTrinoConnectorTest = false + +enableKerberosTest=false +kerberosHmsPort=9883 +kerberosHdfsPort=8820 diff --git a/regression-test/data/external_table_p0/kerberos/test_single_hive_kerberos.out b/regression-test/data/external_table_p0/kerberos/test_single_hive_kerberos.out new file mode 100644 index 000000000000000..95640fecb54fd1b --- /dev/null +++ b/regression-test/data/external_table_p0/kerberos/test_single_hive_kerberos.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +1 a 3.16 cc0 +2 b 41.2 cc1 +3 c 6.2 cc2 +4 d 1.4 cc3 diff --git a/regression-test/data/external_table_p0/kerberos/test_two_hive_kerberos.out b/regression-test/data/external_table_p0/kerberos/test_two_hive_kerberos.out new file mode 100644 index 000000000000000..9415efd787f86d8 --- /dev/null +++ b/regression-test/data/external_table_p0/kerberos/test_two_hive_kerberos.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +1 a 3.16 cc0 +2 b 41.2 cc1 +3 c 6.2 cc2 +4 d 1.4 cc3 + +-- !q02 -- +1 a 3.16 cc0 +2 b 41.2 cc1 +3 c 6.2 cc2 +4 d 1.4 cc3 diff --git a/regression-test/pipeline/external/conf/be.conf b/regression-test/pipeline/external/conf/be.conf index e26929b0dcaf5fa..a7c0713d8eb3adf 100644 --- a/regression-test/pipeline/external/conf/be.conf +++ b/regression-test/pipeline/external/conf/be.conf @@ -62,3 +62,6 @@ enable_debug_log_timeout_secs=0 trino_connector_plugin_dir=/tmp/trino_connector/connectors enable_jvm_monitor = true + +KRB5_CONFIG=/keytabs/krb5.conf +kerberos_krb5_conf_path=/keytabs/krb5.conf diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index 3727b6a9406de07..8eed72816e81389 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -96,3 +96,5 @@ auth_token = 5ff161c3-2c08-4079-b108-26c8850b6598 infodb_support_ext_catalog=true trino_connector_plugin_dir=/tmp/trino_connector/connectors + +KRB5_CONFIG=/keytabs/krb5.conf diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 6b7c451d7cbce7d..97de0530bc98037 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -157,3 +157,8 @@ db2_11_port=50000 // trino-connector catalog test config enableTrinoConnectorTest = true + +// kerberos docker config +enableKerberosTest = true +kerberosHmsPort=9883 +kerberosHdfsPort=8820 diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy new file mode 100644 index 000000000000000..7a0864923f51cb7 --- /dev/null +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { + String enabled = context.config.otherConfigs.get("enableKerberosTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String hms_catalog_name = "test_single_hive_kerberos" + sql """drop catalog if exists hms_kerberos;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://172.31.71.25:9083", + "fs.defaultFS" = "hdfs://172.31.71.25:8020", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.kerberos.principal" = "hive/_HOST@LABS.TERADATA.COM" + ); + """ + sql """ switch hms_kerberos """ + sql """ show databases """ + order_qt_q01 """ select * from hms_kerberos.test_krb_hive_db.test_krb_hive_tbl """ + sql """drop catalog hms_kerberos;""" + + try { + sql """drop catalog if exists hms_kerberos_hadoop_err1;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://172.31.71.25:9083", + "fs.defaultFS" = "hdfs://172.31.71.25:8020", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab" + ); + """ + sql """ switch hms_kerberos_hadoop_err1 """ + sql """ show databases """ + } catch (Exception e) { + logger.info(e.toString()) + // caused by a warning msg if enable sasl on hive but "hive.metastore.sasl.enabled" is not true: + // "set_ugi() not successful, Likely cause: new client talking to old server. Continuing without it." + assertTrue(e.toString().contains("org.apache.thrift.transport.TTransportException: null")) + } + + try { + sql """drop catalog if exists hms_kerberos_hadoop_err2;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err2 + PROPERTIES ( + "type" = "hms", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.uris" = "thrift://172.31.71.25:9083", + "fs.defaultFS" = "hdfs://172.31.71.25:8020" + ); + """ + sql """ switch hms_kerberos_hadoop_err2 """ + sql """ show databases """ + } catch (Exception e) { + // org.apache.thrift.transport.TTransportException: GSS initiate failed + assertTrue(e.toString().contains("Could not connect to meta store using any of the URIs provided. Most recent failure: shade.doris.hive.org.apache.thrift.transport.TTransportException: GSS initiate failed")) + } + + // try { + // sql """ + // CREATE CATALOG IF NOT EXISTS hms_keberos_ccache + // PROPERTIES ( + // "type" = "hms", + // "hive.metastore.uris" = "thrift://172.31.71.25:9083", + // "fs.defaultFS" = "hdfs://172.31.71.25:8020", + // "hadoop.security.authentication" = "kerberos", + // "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + // "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab", + // "hive.metastore.thrift.impersonation.enabled" = true" + // "hive.metastore.client.credential-cache.location" = "hive-presto-master-krbcc" + // ); + // """ + // sql """ switch hms_keberos_ccache """ + // sql """ show databases """ + // } catch (Exception e) { + // logger.error(e.message) + // } + } +} diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy new file mode 100644 index 000000000000000..a3b39d1221a740d --- /dev/null +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { + String enabled = context.config.otherConfigs.get("enableKerberosTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String hms_catalog_name = "test_two_hive_kerberos" + sql """drop catalog if exists ${hms_catalog_name};""" + sql """ + CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://172.31.71.25:9083", + "fs.defaultFS" = "hdfs://172.31.71.25:8020", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.kerberos.principal" = "hive/_HOST@LABS.TERADATA.COM" + ); + """ + + sql """drop catalog if exists other_${hms_catalog_name};""" + sql """ + CREATE CATALOG IF NOT EXISTS other_${hms_catalog_name} + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://172.31.71.26:9083", + "fs.defaultFS" = "hdfs://172.31.71.26:8020", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@OTHERREALM.COM", + "hadoop.kerberos.keytab" = "/keytabs/other-presto-server.keytab", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.kerberos.principal" = "hive/_HOST@OTHERREALM.COM", + "hadoop.security.auth_to_local" ="RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// + RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + DEFAULT" + ); + """ + + // 1. catalogA + sql """switch ${hms_catalog_name};""" + logger.info("switched to catalog " + hms_catalog_name) + sql """ show databases """ + sql """ use test_krb_hive_db """ + order_qt_q01 """ select * from test_krb_hive_db.test_krb_hive_tbl """ + + // 2. catalogB + sql """switch other_${hms_catalog_name};""" + logger.info("switched to other catalog " + hms_catalog_name) + sql """ show databases """ + sql """ use test_krb_hive_db """ + order_qt_q02 """ select * from test_krb_hive_db.test_krb_hive_tbl """ + + sql """drop catalog ${hms_catalog_name};""" + sql """drop catalog other_${hms_catalog_name};""" + } +} From 5c344d8371d3d3e32e9a3be51d3b235c3274ee7a Mon Sep 17 00:00:00 2001 From: zfr95 <87513668+zfr9527@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:14:01 +0800 Subject: [PATCH 26/31] [test](auth)add upgrade and downgrade compatibility test case (#34489) [test](auth)add upgrade and downgrade compatibility test case --- .../test_master_slave_consistency_auth.groovy | 323 ++++++++++++++++++ .../auth_p0/test_select_column_auth.groovy | 126 +++++++ .../suites/auth_up_down_p0/load.groovy | 191 +++++++++++ .../test_grant_revoke_auth.groovy | 75 ++++ 4 files changed, 715 insertions(+) create mode 100644 regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy create mode 100644 regression-test/suites/auth_p0/test_select_column_auth.groovy create mode 100644 regression-test/suites/auth_up_down_p0/load.groovy create mode 100644 regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy diff --git a/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy b/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy new file mode 100644 index 000000000000000..379ea68f3cef0a4 --- /dev/null +++ b/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy @@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite ("test_follower_consistent_auth","p0,auth") { + + def get_follower_ip = { + def result = sql """show frontends;""" + for (int i = 0; i < result.size(); i++) { + if (result[i][7] == "FOLLOWER" && result[i][8] == "false") { + return result[i][1] + } + } + return "null" + } + def switch_ip = get_follower_ip() + if (switch_ip != "null") { + logger.info("switch_ip: " + switch_ip) + def new_jdbc_url = context.config.jdbcUrl.replaceAll(/\/\/[0-9.]+:/, "//${switch_ip}:") + logger.info("new_jdbc_url: " + new_jdbc_url) + + String user = 'test_follower_consistent_user' + String pwd = 'C123_567p' + String dbName = 'test_select_column_auth_db' + String tableName = 'test_select_column_auth_table' + String role = 'test_select_column_auth_role' + String wg = 'test_select_column_auth_wg' + String rg = 'test_select_column_auth_rg' + try_sql("DROP role ${role}") + sql """CREATE ROLE ${role}""" + sql """drop WORKLOAD GROUP if exists '${wg}'""" + sql """CREATE WORKLOAD GROUP "${wg}" + PROPERTIES ( + "cpu_share"="10" + );""" + sql """DROP RESOURCE if exists ${rg}""" + sql """ + CREATE RESOURCE IF NOT EXISTS "${rg}" + PROPERTIES( + "type"="hdfs", + "fs.defaultFS"="127.0.0.1:8120", + "hadoop.username"="hive", + "hadoop.password"="hive", + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ); + """ + try_sql("drop user ${user}") + try_sql """drop table if exists ${dbName}.${tableName}""" + sql """drop database if exists ${dbName}""" + sql """create database ${dbName}""" + sql """ + CREATE TABLE IF NOT EXISTS ${dbName}.`${tableName}` ( + id BIGINT, + username VARCHAR(20) + ) + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """create view ${dbName}.v1 as select * from ${dbName}.${tableName};""" + sql """alter table ${dbName}.${tableName} add rollup rollup1(username)""" + sleep(5 * 1000) + sql """create materialized view mv1 as select username from ${dbName}.${tableName}""" + sleep(5 * 1000) + sql """CREATE MATERIALIZED VIEW ${dbName}.mtmv1 + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ('replication_num' = '1') + AS select username, sum(id) from ${dbName}.${tableName} group by username""" + sql """ + insert into ${dbName}.`${tableName}` values + (1, "111"), + (2, "222"), + (3, "333"); + """ + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """refresh MATERIALIZED VIEW ${dbName}.mtmv1 auto""" + sql """grant select_priv on regression_test to ${user}""" + + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "SHOW CATALOG RECYCLE BIN WHERE NAME = 'test'" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + try { + sql "SHOW CATALOG RECYCLE BIN WHERE NAME = 'test'" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv")) + } + } + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "SHOW DATA" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + try { + sql "SHOW DATA" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv")) + } + } + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + sql """grant select_priv(username) on ${dbName}.${tableName} to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName}" + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + sql "select username from ${dbName}.${tableName}" + } + + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.v1" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + try { + sql "select username from ${dbName}.v1" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + sql """grant select_priv(username) on ${dbName}.v1 to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.v1" + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + sql "select username from ${dbName}.v1" + } + + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.mtmv1" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + try { + sql "select username from ${dbName}.mtmv1" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + sql """grant select_priv(username) on ${dbName}.mtmv1 to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.mtmv1" + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + sql "select username from ${dbName}.mtmv1" + } + + + sql """ADMIN SET FRONTEND CONFIG ('experimental_enable_workload_group' = 'true');""" + sql """set experimental_enable_pipeline_engine = true;""" + + // user + sql """grant select_priv on ${dbName}.${tableName} to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName}" + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + sql "select username from ${dbName}.${tableName}" + } + + sql """revoke select_priv on ${dbName}.${tableName} from ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + + // role + sql """grant select_priv on ${dbName}.${tableName} to ROLE '${role}'""" + sql """grant Load_priv on ${dbName}.${tableName} to ROLE '${role}'""" + sql """grant '${role}' to '${user}'""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName}" + sql """insert into ${dbName}.`${tableName}` values (4, "444")""" + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + sql "select username from ${dbName}.${tableName}" + sql """insert into ${dbName}.`${tableName}` values (4, "444")""" + } + + sql """revoke '${role}' from '${user}'""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + + + // workload group + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """set workload_group = '${wg}';""" + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("USAGE/ADMIN privilege")) + } + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + sql """set workload_group = '${wg}';""" + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("USAGE/ADMIN privilege")) + } + } + sql """GRANT USAGE_PRIV ON WORKLOAD GROUP '${wg}' TO '${user}';""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """set workload_group = '${wg}';""" + sql """select username from ${dbName}.${tableName}""" + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + sql """set workload_group = '${wg}';""" + sql """select username from ${dbName}.${tableName}""" + } + + // resource group + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + def res = sql """SHOW RESOURCES;""" + assertTrue(res == []) + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + def res = sql """SHOW RESOURCES;""" + assertTrue(res == []) + } + sql """GRANT USAGE_PRIV ON RESOURCE ${rg} TO ${user};""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + def res = sql """SHOW RESOURCES;""" + assertTrue(res.size == 10) + } + connect(user=user, password="${pwd}", url=new_jdbc_url) { + def res = sql """SHOW RESOURCES;""" + assertTrue(res.size == 10) + } + + try_sql("DROP USER ${user}") + try_sql("drop workload group if exists ${wg};") + + } + +} diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy b/regression-test/suites/auth_p0/test_select_column_auth.groovy new file mode 100644 index 000000000000000..d81768f725a5c4b --- /dev/null +++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_select_column_auth","p0,auth") { + String user = 'test_select_column_auth_user' + String pwd = 'C123_567p' + String dbName = 'test_select_column_auth_db' + String tableName = 'test_select_column_auth_table' + try_sql("drop user ${user}") + try_sql """drop table if exists ${dbName}.${tableName}""" + sql """drop database if exists ${dbName}""" + + sql """create user '${user}' IDENTIFIED by '${pwd}'""" + + sql """create database ${dbName}""" + sql """ + CREATE TABLE IF NOT EXISTS ${dbName}.`${tableName}` ( + id BIGINT, + username VARCHAR(20) + ) + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """create view ${dbName}.v1 as select * from ${dbName}.${tableName};""" + sql """alter table ${dbName}.${tableName} add rollup rollup1(username)""" + sleep(5 * 1000) + sql """create materialized view mv1 as select username from ${dbName}.${tableName}""" + sleep(5 * 1000) + sql """CREATE MATERIALIZED VIEW ${dbName}.mtmv1 + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ('replication_num' = '1') + AS select username, sum(id) as sum_id from ${dbName}.${tableName} group by username""" + sql """ + insert into ${dbName}.`${tableName}` values + (1, "111"), + (2, "222"), + (3, "333"); + """ + sql """refresh MATERIALIZED VIEW ${dbName}.mtmv1 auto""" + sql """grant select_priv on regression_test to ${user}""" + + // table column + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + sql """grant select_priv(username) on ${dbName}.${tableName} to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName}" + } + + // view column + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.v1" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + sql """grant select_priv(username) on ${dbName}.v1 to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.v1" + } + + // mtmv column + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.mtmv1" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + sql """grant select_priv(username) on ${dbName}.mtmv1 to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.mtmv1" + } + + + // mtmv hit + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "SET enable_materialized_view_rewrite=true" + try { + sql "select username, sum(id) from ${dbName}.${tableName} group by username" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + sql """grant select_priv(username) on ${dbName}.mtmv1 to ${user}""" + sql """grant select_priv(sum_id) on ${dbName}.mtmv1 to ${user}""" + sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}""" + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql "SET enable_materialized_view_rewrite=true" + explain { + sql("""select username, sum(id) from ${dbName}.${tableName} group by username""") + contains "mtmv1(mtmv1)" + } + } + + try_sql("DROP USER ${user}") + +} diff --git a/regression-test/suites/auth_up_down_p0/load.groovy b/regression-test/suites/auth_up_down_p0/load.groovy new file mode 100644 index 000000000000000..fd1405e58e947b0 --- /dev/null +++ b/regression-test/suites/auth_up_down_p0/load.groovy @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_downgrade_prepare_auth","p0,auth") { + + String user1 = 'test_upgrade_downgrade_compatibility_auth_user1' + String user2 = 'test_upgrade_downgrade_compatibility_auth_user2' + String role1 = 'test_upgrade_downgrade_compatibility_auth_role1' + String role2 = 'test_upgrade_downgrade_compatibility_auth_role2' + String pwd = 'C123_567p' + + String dbName = 'test_auth_up_down_db' + String tableName1 = 'test_auth_up_down_table1' + String tableName2 = 'test_auth_up_down_table2' + + String wg1 = 'wg_1' + String wg2 = 'wg_2' + + String rg1 = 'test_up_down_resource_1_hdfs' + String rg2 = 'test_up_down_resource_2_hdfs' + + try_sql("DROP USER ${user1}") + try_sql("DROP USER ${user2}") + try_sql("DROP role ${role1}") + try_sql("DROP role ${role2}") + sql """CREATE USER '${user1}' IDENTIFIED BY '${pwd}'""" + sql """CREATE USER '${user2}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user1}""" + sql """grant select_priv on regression_test to ${user2}""" + + sql """CREATE ROLE ${role1}""" + sql """CREATE ROLE ${role2}""" + + + try_sql """drop table if exists ${dbName}.${tableName1}""" + sql """drop database if exists ${dbName}""" + sql """create database ${dbName}""" + sql """ + CREATE TABLE IF NOT EXISTS ${dbName}.`${tableName1}` ( + id BIGINT, + username VARCHAR(20) + ) + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + try_sql """drop table if exists ${dbName}.${tableName2}""" + sql """ + CREATE TABLE IF NOT EXISTS ${dbName}.`${tableName2}` ( + id BIGINT, + username VARCHAR(20) + ) + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """drop WORKLOAD GROUP if exists '${wg1}'""" + sql """drop WORKLOAD GROUP if exists '${wg2}'""" + sql """CREATE WORKLOAD GROUP "${wg1}" + PROPERTIES ( + "cpu_share"="10" + );""" + sql """CREATE WORKLOAD GROUP "${wg2}" + PROPERTIES ( + "cpu_share"="10" + );""" + + sql """DROP RESOURCE if exists ${rg1}""" + sql """DROP RESOURCE if exists ${rg2}""" + sql """ + CREATE RESOURCE IF NOT EXISTS "${rg1}" + PROPERTIES( + "type"="hdfs", + "fs.defaultFS"="127.0.0.1:8120", + "hadoop.username"="hive", + "hadoop.password"="hive", + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ); + """ + sql """ + CREATE RESOURCE IF NOT EXISTS "${rg2}" + PROPERTIES( + "type"="hdfs", + "fs.defaultFS"="127.0.0.1:8120", + "hadoop.username"="hive", + "hadoop.password"="hive", + "dfs.nameservices" = "my_ha", + "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2", + "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000", + "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000", + "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + ); + """ + + sql """ADMIN SET FRONTEND CONFIG ('experimental_enable_workload_group' = 'true');""" + sql """set experimental_enable_pipeline_engine = true;""" + + // user + sql """grant select_priv on ${dbName}.${tableName1} to ${user1}""" + sql """grant select_priv on ${dbName}.${tableName2} to ${user1}""" + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName1}" + } + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName2}" + } + + sql """revoke select_priv on ${dbName}.${tableName1} from ${user1}""" + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName1}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName2}" + } + + // role + sql """grant select_priv on ${dbName}.${tableName1} to ROLE '${role1}'""" + sql """grant Load_priv on ${dbName}.${tableName1} to ROLE '${role2}'""" + sql """grant '${role1}', '${role2}' to '${user2}'""" + connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName1}" + sql """insert into ${dbName}.`${tableName1}` values (4, "444")""" + } + + sql """revoke '${role1}' from '${user2}'""" + connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName1}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) { + sql """insert into ${dbName}.`${tableName1}` values (5, "555")""" + } + + // workload group + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + sql """set workload_group = '${wg1}';""" + try { + sql "select username from ${dbName}.${tableName2}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("USAGE/ADMIN privilege")) + } + } + sql """GRANT USAGE_PRIV ON WORKLOAD GROUP '${wg1}' TO '${user1}';""" + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + sql """set workload_group = '${wg1}';""" + sql """select username from ${dbName}.${tableName2}""" + } + + // resource group + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + def res = sql """SHOW RESOURCES;""" + assertTrue(res == []) + } + sql """GRANT USAGE_PRIV ON RESOURCE ${rg1} TO ${user1};""" + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + def res = sql """SHOW RESOURCES;""" + assertTrue(res.size == 10) + } +} diff --git a/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy new file mode 100644 index 000000000000000..ff9588b7c481b0c --- /dev/null +++ b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_downgrade_compatibility_auth","p0,auth") { + + sql """ADMIN SET FRONTEND CONFIG ('experimental_enable_workload_group' = 'true');""" + sql """set experimental_enable_pipeline_engine = true;""" + + String user1 = 'test_upgrade_downgrade_compatibility_auth_user1' + String user2 = 'test_upgrade_downgrade_compatibility_auth_user2' + String role1 = 'test_upgrade_downgrade_compatibility_auth_role1' + String role2 = 'test_upgrade_downgrade_compatibility_auth_role2' + String pwd = 'C123_567p' + + String dbName = 'test_auth_up_down_db' + String tableName1 = 'test_auth_up_down_table1' + String tableName2 = 'test_auth_up_down_table2' + + String wg1 = 'wg_1' + String wg2 = 'wg_2' + String rg1 = 'test_up_down_resource_1_hdfs' + String rg2 = 'test_up_down_resource_2_hdfs' + + // user + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName1}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + sql "select username from ${dbName}.${tableName2}" + } + + // role + connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) { + try { + sql "select username from ${dbName}.${tableName1}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("Admin_priv,Select_priv")) + } + } + connect(user=user2, password="${pwd}", url=context.config.jdbcUrl) { + sql """insert into ${dbName}.`${tableName1}` values (5, "555")""" + } + + // workload group + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + sql """set workload_group = '${wg1}';""" + sql """select username from ${dbName}.${tableName2}""" + } + + // resource group + connect(user=user1, password="${pwd}", url=context.config.jdbcUrl) { + def res = sql """SHOW RESOURCES;""" + assertTrue(res.size == 10) + } +} From 9631d3c99fa6658ae695e598b618d076a48ea9f6 Mon Sep 17 00:00:00 2001 From: Pxl Date: Thu, 27 Jun 2024 09:24:30 +0800 Subject: [PATCH 27/31] [Chore](test) add WideInteger unit test from clickhouse (#36752) add WideInteger unit test from clickhouse --- be/test/vec/common/pod_array_test.cpp | 3 + be/test/vec/common/wide_integer_test.cpp | 197 +++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 be/test/vec/common/wide_integer_test.cpp diff --git a/be/test/vec/common/pod_array_test.cpp b/be/test/vec/common/pod_array_test.cpp index c8525d23b97d459..e8b0a67ffa04e26 100644 --- a/be/test/vec/common/pod_array_test.cpp +++ b/be/test/vec/common/pod_array_test.cpp @@ -14,6 +14,9 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +// This file is copied from +// https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/tests/gtest_pod_array.cpp +// and modified by Doris #include "vec/common/pod_array.h" diff --git a/be/test/vec/common/wide_integer_test.cpp b/be/test/vec/common/wide_integer_test.cpp new file mode 100644 index 000000000000000..21dde380da550cb --- /dev/null +++ b/be/test/vec/common/wide_integer_test.cpp @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/tests/gtest_wide_integer.cpp +// and modified by Doris + +#include + +#include "vec/common/uint128.h" +#include "vec/core/types.h" + +namespace doris::vectorized { +TEST(WideInteger, Conversions) { + ASSERT_EQ(UInt64(UInt128(12345678901234567890ULL)), 12345678901234567890ULL); + ASSERT_EQ(UInt64(UInt256(12345678901234567890ULL)), 12345678901234567890ULL); + + ASSERT_EQ(__uint128_t(UInt128(12345678901234567890ULL)), 12345678901234567890ULL); + ASSERT_EQ(__uint128_t(UInt256(12345678901234567890ULL)), 12345678901234567890ULL); + + ASSERT_EQ((UInt64(UInt128(123.456))), 123); + ASSERT_EQ((UInt64(UInt256(123.456))), 123); + + ASSERT_EQ(UInt64(UInt128(123.456F)), 123); + ASSERT_EQ(UInt64(UInt256(123.456F)), 123); + + ASSERT_EQ(Float64(UInt128(1) * 1000000000 * 1000000000 * 1000000000 * 1000000000), 1e36); + + ASSERT_EQ(Float64(UInt256(1) * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000 * + 1000000000 * 1000000000 * 1000000000), + 1e72); +} + +TEST(WideInteger, Arithmetic) { + Int128 minus_one = -1; + Int128 zero = 0; + + zero += -1; + ASSERT_EQ(zero, -1); + ASSERT_EQ(zero, minus_one); + + zero += minus_one; +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + ASSERT_EQ(0, memcmp(&zero, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE", + sizeof(zero))); +#else + ASSERT_EQ(0, memcmp(&zero, "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", + sizeof(zero))); +#endif + zero += 2; + ASSERT_EQ(zero, 0); + + ASSERT_EQ(UInt256(12345678901234567890ULL) * 12345678901234567890ULL / 12345678901234567890ULL, + 12345678901234567890ULL); + ASSERT_EQ(UInt256(12345678901234567890ULL) * UInt256(12345678901234567890ULL) / + 12345678901234567890ULL, + 12345678901234567890ULL); + ASSERT_EQ(UInt256(12345678901234567890ULL) * 12345678901234567890ULL / + UInt256(12345678901234567890ULL), + 12345678901234567890ULL); + ASSERT_EQ(UInt256(12345678901234567890ULL) * 12345678901234567890ULL / 12345678901234567890ULL, + UInt256(12345678901234567890ULL)); + ASSERT_EQ(UInt128(12345678901234567890ULL) * 12345678901234567890ULL / + UInt128(12345678901234567890ULL), + 12345678901234567890ULL); + ASSERT_EQ(UInt256(12345678901234567890ULL) * UInt128(12345678901234567890ULL) / + 12345678901234567890ULL, + 12345678901234567890ULL); + + ASSERT_EQ(Int128(0) + Int32(-1), Int128(-1)); +} + +TEST(WideInteger, DecimalArithmetic) { + Decimal128V3 zero {}; + Decimal32 addend = -1000; + + zero += Decimal128V3(addend); + ASSERT_EQ(zero.value, -1000); + + zero += addend; + ASSERT_EQ(zero.value, -2000); +} + +TEST(WideInteger, FromDouble) { + /// Check that we are being able to convert double to big integer without the help of floating point instructions. + /// (a prototype of a function that we may need) + + double f = -123.456; + UInt64 u; + memcpy(&u, &f, sizeof(f)); + + bool is_negative = u >> 63; + uint16_t exponent = (u >> 52) & (((1ULL << 12) - 1) >> 1); + int16_t normalized_exponent = exponent - 1023; + UInt64 mantissa = u & ((1ULL << 52) - 1); + + // std::cerr << is_negative << ", " << normalized_exponent << ", " << mantissa << "\n"; + + /// x = sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)) + + Int128 res = 0; + + if (normalized_exponent >= 128) { + } else { + res = mantissa; + if (normalized_exponent > 52) { + res <<= (normalized_exponent - 52); + } else { + res >>= (52 - normalized_exponent); + } + + if (normalized_exponent > 0) { + res += Int128(1) << normalized_exponent; + } + } + + if (is_negative) { + res = -res; + } + + ASSERT_EQ(res, -123); +} + +TEST(WideInteger, Shift) { + Int128 x = 1; + + auto y = x << 64; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00", + sizeof(Int128))); +#else + ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00", + sizeof(Int128))); +#endif + auto z = y << 11; + auto a = x << 11; + ASSERT_EQ(a, 2048); + + z >>= 64; + ASSERT_EQ(z, a); + + x = -1; + y = x << 16; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00", + sizeof(Int128))); +#else + ASSERT_EQ(0, memcmp(&y, "\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", + sizeof(Int128))); +#endif + y >>= 16; + ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", + sizeof(Int128))); + + y <<= 64; +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00\x00\x00\x00\x00\x00\x00", + sizeof(Int128))); +#else + ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", + sizeof(Int128))); +#endif + y >>= 32; +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00\x00\x00", + sizeof(Int128))); +#else + ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", + sizeof(Int128))); +#endif + + y <<= 64; +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + sizeof(Int128))); +#else + ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\xFF\xFF", + sizeof(Int128))); +#endif +} + +} // namespace doris::vectorized From 66efefca3c3d4067698833d22e0eccf651eadd2d Mon Sep 17 00:00:00 2001 From: HappenLee Date: Thu, 27 Jun 2024 09:25:15 +0800 Subject: [PATCH 28/31] [Refactor](scanner) remove the unless timer in scanner (#36746) ## Proposed changes remove the unless code in scanner --- be/src/pipeline/exec/scan_operator.cpp | 1 - be/src/pipeline/exec/scan_operator.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index 21f87c68d5d8cf5..2438e402133b80b 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -1298,7 +1298,6 @@ Status ScanLocalState::_init_profile() { _scan_timer = ADD_TIMER(_scanner_profile, "ScannerGetBlockTime"); _scan_cpu_timer = ADD_TIMER(_scanner_profile, "ScannerCpuTime"); - _prefilter_timer = ADD_TIMER(_scanner_profile, "ScannerPrefilterTime"); _convert_block_timer = ADD_TIMER(_scanner_profile, "ScannerConvertBlockTime"); _filter_timer = ADD_TIMER(_scanner_profile, "ScannerFilterTime"); diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h index 6c2c3e80346eb4a..638ec17ccc92700 100644 --- a/be/src/pipeline/exec/scan_operator.h +++ b/be/src/pipeline/exec/scan_operator.h @@ -113,8 +113,6 @@ class ScanLocalStateBase : public PipelineXLocalState<>, public RuntimeFilterCon // time of get block from scanner RuntimeProfile::Counter* _scan_timer = nullptr; RuntimeProfile::Counter* _scan_cpu_timer = nullptr; - // time of prefilter input block from scanner - RuntimeProfile::Counter* _prefilter_timer = nullptr; // time of convert input block to output block from scanner RuntimeProfile::Counter* _convert_block_timer = nullptr; // time of filter output block from scanner From 929f040b1a33843b2be62b533d62f69384a44ec3 Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 27 Jun 2024 09:56:50 +0800 Subject: [PATCH 29/31] [feature](Azure) Implement generate_presigned_url function for Azure object client (#36829) implement generate_presigned_url function for azure object client. The implementation is inspired by the [sample](https://github.com/Azure/azure-sdk-for-cpp/blob/main/sdk/storage/azure-storage-blobs/samples/blob_sas.cpp) code of azure sdk --- be/src/io/fs/azure_obj_storage_client.cpp | 44 ++++++++++++++++++----- be/src/io/fs/azure_obj_storage_client.h | 5 +-- be/src/io/fs/obj_storage_client.h | 12 ++++++- be/src/io/fs/s3_file_system.cpp | 3 +- be/src/io/fs/s3_obj_storage_client.cpp | 21 ++++++----- be/src/io/fs/s3_obj_storage_client.h | 2 +- be/src/util/s3_util.h | 1 + cloud/src/recycler/azure_obj_client.cpp | 20 +++++------ cloud/src/recycler/obj_store_accessor.h | 1 + 9 files changed, 75 insertions(+), 34 deletions(-) diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 1984083f9fbe297..b7ff0a88887ba91 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -28,9 +28,12 @@ #include #include #include +#include #include +#include #include #include +#include #include #include #include @@ -39,6 +42,7 @@ #include "common/logging.h" #include "common/status.h" #include "io/fs/obj_storage_client.h" +#include "util/s3_util.h" using namespace Azure::Storage::Blobs; @@ -52,6 +56,8 @@ auto base64_encode_part_num(int part_num) { return Aws::Utils::HashingUtils::Base64Encode( {reinterpret_cast(&part_num), sizeof(part_num)}); } + +constexpr char SAS_TOKEN_URL_TEMPLATE[] = "https://{}.blob.core.windows.net/{}/{}{}"; } // namespace namespace doris::io { @@ -80,7 +86,7 @@ ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOption LOG_WARNING(msg); return {.status = convert_to_obj_response(Status::InternalError(std::move(msg)))}; } - return {}; + return ObjectStorageResponse::OK(); } struct AzureBatchDeleter { @@ -92,7 +98,7 @@ struct AzureBatchDeleter { } ObjectStorageResponse execute() { if (deferred_resps.empty()) { - return {}; + return ObjectStorageResponse::OK(); } auto resp = do_azure_client_call([&]() { _client->SubmitBatch(_batch); }, _opts); if (resp.status.code != ErrorCode::OK) { @@ -113,7 +119,7 @@ struct AzureBatchDeleter { } } - return {}; + return ObjectStorageResponse::OK(); } private: @@ -126,7 +132,9 @@ struct AzureBatchDeleter { // Azure would do nothing ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { - return {}; + return ObjectStorageUploadResponse { + .resp = ObjectStorageResponse::OK(), + }; } ObjectStorageResponse AzureObjStorageClient::put_object(const ObjectStoragePathOptions& opts, @@ -165,7 +173,9 @@ ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStora }; // clang-format on } - return {}; + return ObjectStorageUploadResponse { + .resp = ObjectStorageResponse::OK(), + }; } ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( @@ -267,7 +277,7 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects(const ObjectStorageP return resp; } } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageResponse AzureObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { @@ -295,7 +305,7 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( if (auto response = deleter.execute(); response.status.code != ErrorCode::OK) { return response; } - return {}; + return ObjectStorageResponse::OK(); }; auto resp = _client->ListBlobs(list_opts); if (auto response = delete_func(resp.Blobs); response.status.code != ErrorCode::OK) { @@ -310,6 +320,24 @@ ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( return response; } } - return {}; + return ObjectStorageResponse::OK(); +} + +std::string AzureObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, + int64_t expiration_secs, + const S3ClientConf& conf) { + Azure::Storage::Sas::BlobSasBuilder sas_builder; + sas_builder.ExpiresOn = + std::chrono::system_clock::now() + std::chrono::seconds(expiration_secs); + sas_builder.BlobContainerName = opts.bucket; + sas_builder.BlobName = opts.key; + sas_builder.Resource = Azure::Storage::Sas::BlobSasResource::Blob; + sas_builder.Protocol = Azure::Storage::Sas::SasProtocol::HttpsOnly; + sas_builder.SetPermissions(Azure::Storage::Sas::BlobSasPermissions::Read); + + std::string sasToken = sas_builder.GenerateSasToken( + Azure::Storage::StorageSharedKeyCredential(conf.ak, conf.sk)); + + return fmt::format(SAS_TOKEN_URL_TEMPLATE, conf.ak, conf.bucket, opts.key, sasToken); } } // namespace doris::io \ No newline at end of file diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h index 7e8678628bd8fcc..ec1a75997b091d5 100644 --- a/be/src/io/fs/azure_obj_storage_client.h +++ b/be/src/io/fs/azure_obj_storage_client.h @@ -51,11 +51,8 @@ class AzureObjStorageClient final : public ObjStorageClient { std::vector objs) override; ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; - // TODO(ByteYue) : to be implemented std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs) override { - return "http://azure.to.be.implenmented"; - }; + int64_t expiration_secs, const S3ClientConf& conf) override; private: std::shared_ptr _client; diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index 8f31b55705bb88d..202b7be2385e691 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -23,6 +23,7 @@ #include "io/fs/path.h" namespace doris { class Status; +struct S3ClientConf; namespace io { // Names are in lexico order. @@ -60,6 +61,14 @@ struct ObjectStorageResponse { ObjectStorageStatus status {}; int http_code {200}; std::string request_id = std::string(); + static ObjectStorageResponse OK() { + // clang-format off + return { + .status { .code = 0, }, + .http_code = 200, + }; + // clang-format on + } }; struct ObjectStorageUploadResponse { @@ -118,7 +127,8 @@ class ObjStorageClient { const ObjectStoragePathOptions& opts) = 0; // Return a presigned URL for users to access the object virtual std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs) = 0; + int64_t expiration_secs, + const S3ClientConf& conf) = 0; }; } // namespace io } // namespace doris diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 0a6048592b79082..2cc82d2da0f6a95 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -441,7 +441,8 @@ std::string S3FileSystem::generate_presigned_url(const Path& path, int64_t expir } else { client = _client->get(); } - return client->generate_presigned_url({.bucket = _bucket, .key = key}, expiration_secs); + return client->generate_presigned_url({.bucket = _bucket, .key = key}, expiration_secs, + _client->s3_client_conf()); } } // namespace doris::io diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index c6cda174833c14a..524a8d33ccef3a3 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -133,7 +133,7 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti static_cast(response.GetError().GetResponseCode()), response.GetError().GetRequestId()}; } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, @@ -211,7 +211,7 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( static_cast(complete_outcome.GetError().GetResponseCode()), complete_outcome.GetError().GetRequestId()}; } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { @@ -233,7 +233,9 @@ ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePat static_cast(outcome.GetError().GetResponseCode()), outcome.GetError().GetRequestId()}}; } - return {}; + return ObjectStorageHeadResponse { + .resp = ObjectStorageResponse::OK(), + }; } ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOptions& opts, @@ -259,7 +261,7 @@ ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOpti Status::InternalError("failed to read from {}(bytes read: {}, bytes req: {})", opts.path.native(), *size_return, bytes_read))}; } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, @@ -292,7 +294,7 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp is_trucated = outcome.GetResult().GetIsTruncated(); request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); } while (is_trucated); - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePathOptions& opts, @@ -322,7 +324,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath return {convert_to_obj_response(Status::InternalError("failed to delete object {}: {}", e.GetKey(), e.GetMessage()))}; } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { @@ -333,7 +335,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathO auto outcome = _client->DeleteObject(request); if (outcome.IsSuccess() || outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {}; + return ObjectStorageResponse::OK(); } return {convert_to_obj_response(s3fs_error(outcome.GetError(), fmt::format("failed to delete file {}", opts.key))), @@ -389,11 +391,12 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( is_trucated = result.GetIsTruncated(); request.SetContinuationToken(result.GetNextContinuationToken()); } while (is_trucated); - return {}; + return ObjectStorageResponse::OK(); } std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs) { + int64_t expiration_secs, + const S3ClientConf&) { return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, expiration_secs); } diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index ebc81b81b4052da..45294226594d813 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -54,7 +54,7 @@ class S3ObjStorageClient final : public ObjStorageClient { ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs) override; + int64_t expiration_secs, const S3ClientConf&) override; private: std::shared_ptr _client; diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 5b0e9c092001841..587140a7acca751 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -107,6 +107,7 @@ struct S3ClientConf { hash_code ^= request_timeout_ms; hash_code ^= connect_timeout_ms; hash_code ^= use_virtual_addressing; + hash_code ^= static_cast(provider); return hash_code; } diff --git a/cloud/src/recycler/azure_obj_client.cpp b/cloud/src/recycler/azure_obj_client.cpp index 0d5e3f54827a587..b5cd3e76d8ec206 100644 --- a/cloud/src/recycler/azure_obj_client.cpp +++ b/cloud/src/recycler/azure_obj_client.cpp @@ -59,7 +59,7 @@ ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOption LOG_WARNING(msg); return {-1, std::move(msg)}; } - return {}; + return ObjectStorageResponse::OK(); } struct AzureBatchDeleter { @@ -71,7 +71,7 @@ struct AzureBatchDeleter { } ObjectStorageResponse execute() { if (deferred_resps.empty()) { - return {}; + return ObjectStorageResponse::OK(); } auto resp = do_azure_client_call([&]() { _client->SubmitBatch(_batch); }, _opts); if (resp.ret != 0) { @@ -93,7 +93,7 @@ struct AzureBatchDeleter { } } - return {}; + return ObjectStorageResponse::OK(); } private: @@ -117,7 +117,7 @@ ObjectStorageResponse AzureObjClient::head_object(const ObjectStoragePathOptions try { Models::BlobProperties properties = _client->GetBlockBlobClient(opts.key).GetProperties().Value; - return {}; + return ObjectStorageResponse::OK(); } catch (Azure::Storage::StorageException& e) { if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { return {1}; @@ -165,7 +165,7 @@ ObjectStorageResponse AzureObjClient::delete_objects(const ObjectStoragePathOpti std::vector objs) { if (objs.empty()) { LOG_INFO("No objects to delete").tag("endpoint", opts.endpoint).tag("bucket", opts.bucket); - return {}; + return ObjectStorageResponse::OK(); } // TODO(ByteYue) : use range to adate this code when compiler is ready // auto chunkedView = objs | std::views::chunk(BlobBatchMaxOperations); @@ -185,7 +185,7 @@ ObjectStorageResponse AzureObjClient::delete_objects(const ObjectStoragePathOpti return resp; } } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageResponse AzureObjClient::delete_object(const ObjectStoragePathOptions& opts) { @@ -213,7 +213,7 @@ ObjectStorageResponse AzureObjClient::delete_objects_recursively( if (auto response = deleter.execute(); response.ret != 0) { return response; } - return {}; + return ObjectStorageResponse::OK(); }; auto resp = _client->ListBlobs(list_opts); if (auto response = delete_func(resp.Blobs); response.ret != 0) { @@ -227,7 +227,7 @@ ObjectStorageResponse AzureObjClient::delete_objects_recursively( return response; } } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageResponse AzureObjClient::delete_expired(const ObjectStorageDeleteExpiredOptions& opts, @@ -246,7 +246,7 @@ ObjectStorageResponse AzureObjClient::delete_expired(const ObjectStorageDeleteEx if (auto response = deleter.execute(); response.ret != 0) { return response; } - return {}; + return ObjectStorageResponse::OK(); }; auto resp = _client->ListBlobs(list_opts); auto response = delete_func(resp.Blobs); @@ -261,7 +261,7 @@ ObjectStorageResponse AzureObjClient::delete_expired(const ObjectStorageDeleteEx return response; } } - return {}; + return ObjectStorageResponse::OK(); } ObjectStorageResponse AzureObjClient::get_life_cycle(const ObjectStoragePathOptions& opts, diff --git a/cloud/src/recycler/obj_store_accessor.h b/cloud/src/recycler/obj_store_accessor.h index 0238a317880d700..dec87fb1a3ecec7 100644 --- a/cloud/src/recycler/obj_store_accessor.h +++ b/cloud/src/recycler/obj_store_accessor.h @@ -97,6 +97,7 @@ struct ObjectStorageResponse { int ret {0}; // To unify the error handle logic with BE, we'd better use the same error code as BE // clang-format on std::string error_msg; + static ObjectStorageResponse OK() { return {0}; } }; // wrapper class owned by concret fs From 233b0b5dd9d382102fb021573132872675f17182 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Thu, 27 Jun 2024 10:54:32 +0800 Subject: [PATCH 30/31] [fix](bitmap) incorrect type of BitmapValue with fastunion (#36834) --- be/src/util/bitmap_value.h | 5 +++-- be/test/util/bitmap_value_test.cpp | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index c7befb243dbb819..ac5e826e1fee665 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1307,7 +1307,6 @@ class BitmapValue { case SINGLE: { _set.insert(_sv); _type = SET; - _convert_to_bitmap_if_need(); break; } case BITMAP: @@ -1318,10 +1317,12 @@ class BitmapValue { _type = BITMAP; break; case SET: { - _convert_to_bitmap_if_need(); break; } } + if (_type == SET) { + _convert_to_bitmap_if_need(); + } } if (_type == EMPTY && single_values.size() == 1) { diff --git a/be/test/util/bitmap_value_test.cpp b/be/test/util/bitmap_value_test.cpp index a8fb9bb2b5e4816..2c08161450ecc34 100644 --- a/be/test/util/bitmap_value_test.cpp +++ b/be/test/util/bitmap_value_test.cpp @@ -871,6 +871,29 @@ TEST(BitmapValueTest, bitmap_union) { EXPECT_EQ(3, bitmap3.cardinality()); bitmap3.fastunion({&bitmap}); EXPECT_EQ(5, bitmap3.cardinality()); + + const auto old_config = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = true; + BitmapValue bitmap4; // empty + + BitmapValue bitmap_set1; + BitmapValue bitmap_set2; + BitmapValue bitmap_set3; + + const int set_data1[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + bitmap_set1.add_many(set_data1, 15); + + const int set_data2[] = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; + bitmap_set2.add_many(set_data2, 15); + + const int set_data3[] = {31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45}; + bitmap_set3.add_many(set_data3, 15); + + bitmap4.fastunion({&bitmap_set1, &bitmap_set2, &bitmap_set3}); + + EXPECT_EQ(bitmap4.cardinality(), 45); + EXPECT_EQ(bitmap4.get_type_code(), BitmapTypeCode::BITMAP32); + config::enable_set_in_bitmap_value = old_config; } TEST(BitmapValueTest, bitmap_intersect) { From 23b0b7b28d422f2f891d1f52d3fba3bdb8f3bd7e Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Thu, 27 Jun 2024 11:15:32 +0800 Subject: [PATCH 31/31] [fix](tvf) Partition columns in CTAS need to be compatible with the STRING type of external tables/TVF (#35489) Partition columns in CTAS need to be compatible with the STRING type of external tables/TVF 1. For TVF queries with partition columns in the file path, map these partition columns to the `VARCHAR(65533)` type. (Previously, they were of `STRING` type.) 3. If you use the `STRING` type of an external table as a partition column, CTAS will automatically convert it to `VARCHAR(65535)` type." --- .../doris/analysis/DistributionDesc.java | 8 + .../doris/analysis/HashDistributionDesc.java | 1 + .../apache/doris/analysis/PartitionDesc.java | 4 + .../analysis/RandomDistributionDesc.java | 5 + .../doris/datasource/InternalCatalog.java | 14 +- .../hive/ThriftHMSCachedClient.java | 2 +- .../nereids/parser/PartitionTableInfo.java | 48 ++- .../plans/commands/CreateTableCommand.java | 17 +- .../plans/commands/info/CreateTableInfo.java | 8 + .../commands/info/DistributionDescriptor.java | 4 + .../ExternalFileTableValuedFunction.java | 2 +- .../hive/write/test_hive_ctas_to_doris.out | 37 ++ .../tvf/test_ctas_with_hdfs.out | 383 ++++++++++++++++++ .../hive/ddl/test_hive_ddl.groovy | 4 +- .../hive/write/test_hive_ctas_to_doris.groovy | 100 +++++ .../tvf/test_ctas_with_hdfs.groovy | 129 ++++++ 16 files changed, 735 insertions(+), 31 deletions(-) create mode 100644 regression-test/data/external_table_p0/hive/write/test_hive_ctas_to_doris.out create mode 100644 regression-test/data/external_table_p0/tvf/test_ctas_with_hdfs.out create mode 100644 regression-test/suites/external_table_p0/hive/write/test_hive_ctas_to_doris.groovy create mode 100644 regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DistributionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DistributionDesc.java index c9bf49b116454d7..83205deef2fdffb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DistributionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DistributionDesc.java @@ -65,4 +65,12 @@ public String toSql() { public DistributionInfo toDistributionInfo(List columns) throws DdlException { throw new NotImplementedException("toDistributionInfo not implemented"); } + + public List getDistributionColumnNames() { + throw new NotImplementedException("getDistributionColumnNames not implemented"); + } + + public boolean inDistributionColumns(String columnName) { + return getDistributionColumnNames() != null && getDistributionColumnNames().contains(columnName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java index 374bf10bdbfd708..51b1454110c997b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java @@ -47,6 +47,7 @@ public HashDistributionDesc(int numBucket, boolean autoBucket, List dist this.distributionColumnNames = distributionColumnNames; } + @Override public List getDistributionColumnNames() { return distributionColumnNames; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java index 81bec358884a68c..eb22ff06e444698 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -285,4 +285,8 @@ public PartitionInfo toPartitionInfo(List schema, Map part throws DdlException, AnalysisException { throw new NotImplementedException("toPartitionInfo not implemented"); } + + public boolean inIdentifierPartitions(String colName) { + return partitionColNames != null && partitionColNames.contains(colName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RandomDistributionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RandomDistributionDesc.java index 3b89dfaff7a2a3a..4ffc24fdb19a2c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RandomDistributionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RandomDistributionDesc.java @@ -62,4 +62,9 @@ public DistributionInfo toDistributionInfo(List columns) { RandomDistributionInfo randomDistributionInfo = new RandomDistributionInfo(numBucket, autoBucket); return randomDistributionInfo; } + + @Override + public List getDistributionColumnNames() { + return null; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index f3642700c846d30..68d7db6256cce9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1325,7 +1325,19 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio if (resultExpr.getSrcSlotRef() != null && resultExpr.getSrcSlotRef().getTable() != null && !resultExpr.getSrcSlotRef().getTable().isManagedTable()) { - typeDef = new TypeDef(ScalarType.createStringType()); + if (createTableStmt.getPartitionDesc().inIdentifierPartitions( + resultExpr.getSrcSlotRef().getColumnName()) + || (createTableStmt.getDistributionDesc() != null + && createTableStmt.getDistributionDesc().inDistributionColumns( + resultExpr.getSrcSlotRef().getColumnName()))) { + // String type can not be used in partition/distributed column + // so we replace it to varchar + if (resultType.getPrimitiveType() == PrimitiveType.STRING) { + typeDef = new TypeDef(ScalarType.createVarchar(ScalarType.MAX_VARCHAR_LENGTH)); + } + } else { + typeDef = new TypeDef(ScalarType.createStringType()); + } } } else if (resultType.isDecimalV2() && resultType.equals(ScalarType.DECIMALV2)) { typeDef = new TypeDef(ScalarType.createDecimalType(27, 9)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java index 0acf8893267912e..bc4da6762caaef5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java @@ -206,7 +206,7 @@ public void createTable(TableMetadata tbl, boolean ignoreIfExists) { throw e; } } catch (Exception e) { - throw new HMSClientException("failed to create database from hms client", e); + throw new HMSClientException("failed to create table from hms client", e); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java index 38601ba6d8045a4..a68ddcdf87ac24f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java @@ -63,10 +63,14 @@ public class PartitionTableInfo { null); private boolean isAutoPartition; + // for PartitionType private String partitionType; - private List partitionColumns; private List partitionDefs; + // save all list partition expressions, including identifier and function private List partitionList; + // save identifier expressions in partitionList, + // facilitates subsequent verification process + private List identifierPartitionColumns; /** * struct for partition definition @@ -85,7 +89,7 @@ public PartitionTableInfo( this.partitionDefs = partitionDefs; this.partitionList = partitionFields; if (this.partitionList != null) { - this.partitionColumns = this.partitionList.stream() + this.identifierPartitionColumns = this.partitionList.stream() .filter(UnboundSlot.class::isInstance) .map(partition -> ((UnboundSlot) partition).getName()) .collect(Collectors.toList()); @@ -100,10 +104,6 @@ public String getPartitionType() { return partitionType; } - public List getPartitionColumns() { - return partitionColumns; - } - /** * check partitions types. */ @@ -169,16 +169,16 @@ public void validatePartitionInfo( boolean isEnableMergeOnWrite, boolean isExternal) { - if (partitionColumns != null) { + if (identifierPartitionColumns != null) { - if (partitionColumns.size() != partitionList.size()) { + if (identifierPartitionColumns.size() != partitionList.size()) { if (!isExternal && partitionType.equalsIgnoreCase(PartitionType.LIST.name())) { throw new AnalysisException("internal catalog does not support functions in 'LIST' partition"); } isAutoPartition = true; } - partitionColumns.forEach(p -> { + identifierPartitionColumns.forEach(p -> { if (!columnMap.containsKey(p)) { throw new AnalysisException( String.format("partition key %s is not exists", p)); @@ -187,7 +187,7 @@ public void validatePartitionInfo( }); Set partitionColumnSets = Sets.newHashSet(); - List duplicatesKeys = partitionColumns.stream() + List duplicatesKeys = identifierPartitionColumns.stream() .filter(c -> !partitionColumnSets.add(c)).collect(Collectors.toList()); if (!duplicatesKeys.isEmpty()) { throw new AnalysisException( @@ -199,16 +199,16 @@ public void validatePartitionInfo( // 2. The partition field must be at the end of the schema // 3. The order of partition fields in the schema // must be consistent with the order defined in `PARTITIONED BY LIST()` - if (partitionColumns.size() == columns.size()) { + if (identifierPartitionColumns.size() == columns.size()) { throw new AnalysisException("Cannot set all columns as partitioning columns."); } List partitionInSchema = columns.subList( - columns.size() - partitionColumns.size(), columns.size()); - if (partitionInSchema.stream().anyMatch(p -> !partitionColumns.contains(p.getName()))) { + columns.size() - identifierPartitionColumns.size(), columns.size()); + if (partitionInSchema.stream().anyMatch(p -> !identifierPartitionColumns.contains(p.getName()))) { throw new AnalysisException("The partition field must be at the end of the schema."); } for (int i = 0; i < partitionInSchema.size(); i++) { - if (!partitionInSchema.get(i).getName().equals(partitionColumns.get(i))) { + if (!partitionInSchema.get(i).getName().equals(identifierPartitionColumns.get(i))) { throw new AnalysisException("The order of partition fields in the schema " + "must be consistent with the order defined in `PARTITIONED BY LIST()`"); } @@ -234,7 +234,7 @@ public void validatePartitionInfo( partitionNames.add(partitionName); } partitionDefs.forEach(p -> { - p.setPartitionTypes(partitionColumns.stream() + p.setPartitionTypes(identifierPartitionColumns.stream() .map(s -> columnMap.get(s).getType()).collect(Collectors.toList())); p.validate(Maps.newHashMap(properties)); }); @@ -269,18 +269,18 @@ public PartitionDesc convertToPartitionDesc(boolean isExternal) { try { ArrayList exprs = convertToLegacyAutoPartitionExprs(partitionList); - // here we have already extracted partitionColumns + // here we have already extracted identifierPartitionColumns if (partitionType.equals(PartitionType.RANGE.name())) { if (isAutoPartition) { - partitionDesc = new RangePartitionDesc(exprs, partitionColumns, partitionDescs); + partitionDesc = new RangePartitionDesc(exprs, identifierPartitionColumns, partitionDescs); } else { - partitionDesc = new RangePartitionDesc(partitionColumns, partitionDescs); + partitionDesc = new RangePartitionDesc(identifierPartitionColumns, partitionDescs); } } else { if (isAutoPartition) { - partitionDesc = new ListPartitionDesc(exprs, partitionColumns, partitionDescs); + partitionDesc = new ListPartitionDesc(exprs, identifierPartitionColumns, partitionDescs); } else { - partitionDesc = new ListPartitionDesc(partitionColumns, partitionDescs); + partitionDesc = new ListPartitionDesc(identifierPartitionColumns, partitionDescs); } } } catch (Exception e) { @@ -319,7 +319,7 @@ private static List convertToLegacyArguments(List children) { } /** - * Get column names and put in partitionColumns + * Get column names and put in identifierPartitionColumns */ public void extractPartitionColumns() throws AnalysisException { if (partitionList == null) { @@ -327,10 +327,14 @@ public void extractPartitionColumns() throws AnalysisException { } ArrayList exprs = convertToLegacyAutoPartitionExprs(partitionList); try { - partitionColumns = PartitionDesc.getColNamesFromExpr(exprs, + identifierPartitionColumns = PartitionDesc.getColNamesFromExpr(exprs, partitionType.equalsIgnoreCase(PartitionType.LIST.name()), isAutoPartition); } catch (Exception e) { throw new AnalysisException(e.getMessage(), e.getCause()); } } + + public boolean inIdentifierPartitions(String columnName) { + return identifierPartitionColumns != null && identifierPartitionColumns.contains(columnName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 0594cfb68332992..7df93d8b9a9db53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -128,10 +128,19 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, DecimalV2Type.class, DecimalV2Type.SYSTEM_DEFAULT); if (s.isColumnFromTable()) { - if (!((SlotReference) s).getTable().isPresent() - || !((SlotReference) s).getTable().get().isManagedTable()) { - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - CharacterType.class, StringType.INSTANCE); + if ((!((SlotReference) s).getTable().isPresent() + || !((SlotReference) s).getTable().get().isManagedTable())) { + if (createTableInfo.getPartitionTableInfo().inIdentifierPartitions(s.getName()) + || (createTableInfo.getDistribution() != null + && createTableInfo.getDistribution().inDistributionColumns(s.getName()))) { + // String type can not be used in partition/distributed column + // so we replace it to varchar + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + StringType.class, VarcharType.MAX_VARCHAR_TYPE); + } else { + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharacterType.class, StringType.INSTANCE); + } } } else { dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 5c83252ace06905..38b576af17511dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -1044,5 +1044,13 @@ public boolean isGeneratedColumn() { return isGeneratedColumn; } } + + public PartitionTableInfo getPartitionTableInfo() { + return partitionTableInfo; + } + + public DistributionDescriptor getDistribution() { + return distribution; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java index 0b86bf74c586d10..86b158a9e06ad16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java @@ -107,4 +107,8 @@ public DistributionDesc translateToCatalogStyle() { } return new RandomDistributionDesc(bucketNum, isAutoBucket); } + + public boolean inDistributionColumns(String columnName) { + return cols != null && cols.contains(columnName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 10c57f6865567bf..518552bff4abc2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -465,7 +465,7 @@ private void fillColumns(InternalService.PFetchTableSchemaResult result) { // HACK(tsy): path columns are all treated as STRING type now, after BE supports reading all columns // types by all format readers from file meta, maybe reading path columns types from BE then. for (String colName : pathPartitionKeys) { - columns.add(new Column(colName, Type.STRING, false)); + columns.add(new Column(colName, ScalarType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH), false)); } } diff --git a/regression-test/data/external_table_p0/hive/write/test_hive_ctas_to_doris.out b/regression-test/data/external_table_p0/hive/write/test_hive_ctas_to_doris.out new file mode 100644 index 000000000000000..656b47258aaf1fd --- /dev/null +++ b/regression-test/data/external_table_p0/hive/write/test_hive_ctas_to_doris.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +65540 65533 4 + +-- !q02 -- +id INT Yes true \N +str1 TEXT Yes true \N +str2 TEXT Yes true \N +str3 TEXT Yes true \N + +-- !q03 -- +65540 65533 4 + +-- !q04 -- +id INT Yes true \N +str1 TEXT Yes false \N NONE +str2 TEXT Yes false \N NONE +str3 VARCHAR(65533) Yes false \N NONE + +-- !q05 -- +65540 65533 4 + +-- !q06 -- +id INT Yes true \N +str1 TEXT Yes false \N NONE +str2 VARCHAR(65533) Yes false \N NONE +str3 TEXT Yes false \N NONE + +-- !q07 -- +65540 65533 4 + +-- !q08 -- +id INT Yes true \N +str1 TEXT Yes false \N NONE +str2 TEXT Yes false \N NONE +str3 VARCHAR(65533) Yes false \N NONE + diff --git a/regression-test/data/external_table_p0/tvf/test_ctas_with_hdfs.out b/regression-test/data/external_table_p0/tvf/test_ctas_with_hdfs.out new file mode 100644 index 000000000000000..953a99608ca7c1a --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_ctas_with_hdfs.out @@ -0,0 +1,383 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc -- +bigint_col BIGINT Yes false \N NONE +binary_col TEXT Yes false \N NONE +boolean_col BOOLEAN Yes false \N NONE +char_col TEXT Yes false \N NONE +date_col DATE Yes false \N NONE +decimal_col DECIMAL(12, 4) Yes false \N NONE +double_col DOUBLE Yes false \N NONE +float_col FLOAT Yes false \N NONE +int_col INT Yes true \N +list_double_col ARRAY Yes false \N NONE +list_string_col ARRAY Yes false \N NONE +p1_col VARCHAR(65533) No false \N NONE +p2_col VARCHAR(65533) No false \N NONE +smallint_col SMALLINT Yes true \N +string_col TEXT Yes false \N NONE +timestamp_col DATETIME(6) Yes false \N NONE +tinyint_col TINYINT Yes true \N +varchar_col TEXT Yes false \N NONE + +-- !select -- +1 \N \N 757403305318104467 false 3.26199968E8 1.0049111235672792E17 Consolidate iron breakfast inhibit obesity mount hearing. Limitation bite sibling creation between sound. Plus1 layer injury favourable detain. Learn pronounced entrepreneur personnel wool strive. Pose curiosity spite absolutely combination right. \N 2022-08-11T10:09:31 996888.8617 desktops bigint_col 2015-08-24 [5.084045411017597e+17, 3.942856911182207e+17, 8.38109720690003e+17, 5.0079271855467546e+17] ["NRcqedH", "JIkT", "JXw", "JLvj"] desktops bigint_col +1 \N 107386238 108422142707872166 true 8.3058278E8 3.7039169232624294E17 Cut god fade script breakfast delete. Reasoning enthusiast column genetic tolerate cop school. false merge onto 2022-08-18T16:59:24 549352.8096 desktops bigint_col 2015-10-23 [] ["URknu", "WpuINXTasndUKtb", "tSIMVWOtOQy", "nmFV", "QRnFVjUex"] desktops bigint_col +1 \N 653934097 947581034711672436 false 3.82426912E8 \N Sorry shortage prevalence dismissal probability warrior motorist. Discover cancer gate spell. Initially investigator opt cannot colourful production. Shoot capable concentration jail utilize. patent disastrous 2022-08-10T06:28:28 534581.4191 desktops bigint_col 2012-10-22 [9.955149870448799e+17, 5.458963138726759e+17] ["kThacn", "iqAYojfNU", "OmajtKJm", "qfWesYxCEJWqTS", "ZrwOMXcp", "DPeldCDbfKlJuGZKYwTK"] desktops bigint_col +1 11 809290623 686793101332632 true 4.71326496E8 3.3112444749135328E16 Dream biology decline modification councillor large-scale stair. Perfectly possess abundance. Constitution carbon amazed declare opposed lesser supervisor. Accountant furious consolidate atmosphere pin direction this. Creator illustrate yell viewer ankle. disclosure old appetite \N 231822.0509 desktops bigint_col 2022-02-23 [] \N desktops bigint_col +1 12 720480298 439676960593506542 true 8.7667955E8 \N Revision flying spelling steer vicious apparent. Tendency follow must adhere reply festival. half title stupid virtual 2022-08-26T05:05:27 644585.1705 desktops bigint_col \N [5.799499253822285e+17, 6.977374510334419e+17, 2.8792573287467094e+17] [] desktops bigint_col +1 12 872242197 39372924164800285 true 4.02162976E8 5.4119588683596512E17 Bulk fundamental disagree bench past acknowledge. Mercy toxic sing. might chef \N 388501.8604 desktops bigint_col 2022-08-29 [] [null, "hYetKKEDJm"] desktops bigint_col +1 14 726053880 161015669819134929 false 4.9883168E8 4.3323385209889466E17 Team always illusion advantage could. Instrumental stability seven manager care declare. Jewellery navigation bee creature violence bathroom or. Routine identity application presume him. Pan emerge temporarily. counter consciousness study 2022-08-08T13:52:40 366074.9783 desktops bigint_col 2013-02-09 [1.7713806707666048e+17, 3.7383671495565165e+17, 6.259419911350532e+17, 2.5287201471209952e+17] ["vaSfuLXpBIzKBLnaBd", "DXbG", "OaCJCGdl"] desktops bigint_col +1 15 98067009 107002720698930270 true 1.01148912E8 5.7735895489617459E17 Single misleading neutral unknown footage above kind. Walk insect harvest hair suspicion traffic. Slap endure text instrument naval again. \N 2022-08-13T01:39:09 332059.6117 desktops bigint_col 2013-04-09 [7.035787745720701e+17] ["wBJJPblSffQaeWC", "OwhqU", "EqotFomTShNObQcm", null, "nMRD", null, "yhtbbFbanWuSmjycpgLj", "ALyVKaolMG", null] desktops bigint_col +1 20 86450486 128585355387472229 true 3.85727264E8 8.0578324815203955E17 Renew review concentration child pronounce nod emergence. Due travel letter red. private each rocket hers 2022-08-11T04:59:32 644172.2438 desktops bigint_col 2017-12-20 [] [null, "BJWAHVFXmUmTgeJ", "yGZpeeuEAKC"] desktops bigint_col +1 20 907476179 51144570531813048 \N 1.14442112E8 6.3573879782317082E17 Guilt decide electronic expectation potential outdoors. Magnitude unable furious. healthy bike controversy 2022-09-03T09:08:57 368422.0700 desktops bigint_col 2021-06-18 [9.10700912260912e+17, 4.405327840725512e+17] [] desktops bigint_col +1 28 837555130 230916677115824233 true 5.4569421E8 2.70471322007323488E17 Cousin artificial abandon custody beam provision. Chemical abroad human. Arms young see. handy initial neat navigation 2022-09-03T17:55:12 524469.0578 desktops bigint_col 2018-09-04 [3.855014125096763e+17, 2.1039308993257866e+17, 9.31498606282679e+17] ["lgtZYBShnyC", "nRQjgF", "aCVIMQpZuYqOig", "lbVNNHeRK", "HsOO", "IJYFhh", "MDthVyeZpCjF", "sMnbIiEFr", "CyLO", "UAWGAZiQDzkpQBYD"] desktops bigint_col +1 28 838335879 382950133409826292 true 6.2795968E8 3.7588441507785466E17 Evaluate locate guitar frightened. Premier warrior organized buy eat. About die rock creep. civilization discretion desert connect firework 2022-09-02T22:27:57 479988.5725 desktops bigint_col 2020-12-16 [7.804613874622225e+17, 1.8854381373872086e+17] ["Mfwddzogb", "LGdzf", "gEQy"] desktops bigint_col +1 31 \N 741341588454293823 true 3.86187136E8 4.6610951470341754E17 Aspiration deployment tough make-up. Listing bride hole revision criticize ray sponsorship. Former whereby vanish strain typically. Transition lost erect settle provoke fate. Action atrocity wildlife weird helpful inclined. vary matching \N 389141.8253 desktops bigint_col 2019-04-30 [2.6031378443226595e+17, 9.016824926976108e+17, 5.316609472515438e+17, 3.669648424112748e+17] [null, "EEXcpNyLcwWMyfI"] desktops bigint_col +1 36 260311723 663342854057084267 false 7.9524992E8 \N Surround characterize wonder. Lady inner cost. Succession clerk increase religious governor squad. Reconstruction commission immediate bit hurt debate. variable contractor female pastor 2022-09-03T19:12:06 947575.7788 desktops bigint_col 2020-12-23 [] ["qiqjac", "SRZCFcqIelw", "rPDODktyX", "QYIZgxOhzLpMIKkzYik", "yfrsvMThiRjyFUypbZ", "WaVFzUSxW", "PRPQuClAUEdqbI", "WFA", "vbQoqNFLfaWRFVcrDX", "lfW"] desktops bigint_col +1 40 706858385 86990101748729070 true 5.4196077E8 6.9602303815701107E17 Resemble detection wood give launch distinction. Per attention deliberate mum. Modification segment foreigner complement conflict. Travel historic over advertisement. Baby recover soft satisfaction relatively. litter alone 2022-08-08T20:38:09 922369.7339 desktops bigint_col 2014-11-13 [5.85367882988869e+17, 6.078596726647018e+17] ["mDBdSwBdKdvhBaComWk", null, "rpVrUUrGMTtJ", "RYTzZRieSBWkDMiJF", "FapEEnbaHD", null, "Gxqzfggamjd", "DMd", "SFMJCivDFFiXOak", "reEcSR"] desktops bigint_col +1 56 563643370 \N false 7.2356896E8 8.2856413152776218E17 \N since super fare wipe 2022-08-14T17:47:58 105736.4432 desktops bigint_col 2012-10-25 [9.610180903194888e+17, 7.288131679313782e+17, 8.355045939461916e+17, 3.4987201052222304e+17] ["hoZwRj", "cMhAJCpHrUZQ", "ocYAxs", "gGVqSQewuauBCUkvzr", "yckCKXxVhrdJp", "eZCHD"] desktops bigint_col +1 58 968282995 675983799610076771 true 1.7376632E8 1.67326199513268896E17 \N \N 2022-08-11T17:23:24 136928.4456 desktops bigint_col 2016-03-24 [6.426187514709601e+17, 8.332563760183391e+17, 5.7197719695391334e+17] ["rpBteuiLfmGcIIqP", "vBeWoaqiXYckauaJ", "kUepRQOLvwqc", "kbIBPMSLEgpaACkdhI", "nBQuR", null, "RPFYYfTTHqI", "iflOfcPV", "qMIr"] desktops bigint_col +1 63 \N 122703346364718870 false 9.4316915E8 7.1076074324903078E17 Found illegal battery exam originally depressing profound. Too despite pot old go apparatus description. Goal consensus update take. beneficial lunch troop restrict 2022-08-26T10:00:47 339452.6832 desktops bigint_col 2017-10-07 [9.981882027272243e+17, 6.346875650940211e+17] \N desktops bigint_col +1 68 317879314 979790364239098905 true 2.34733152E8 9.8859784125654656E16 Procedure can2 dialogue. Why promise wing carefully functional. Enthusiast active chop reality eventually thirsty climb. Crystal rule nearby pop disagree picture. special phone immense \N 517206.9006 desktops bigint_col 2019-07-10 [] \N desktops bigint_col +1 73 \N 41268263353414053 true 4.40259392E8 \N Accumulate expand deadly danger scheme acute privatization. Touch county independence. Demand medication estate high-profile amid explosion. encompass icon end 2022-08-14T04:16:56 544035.6731 desktops bigint_col 2013-05-15 [3.104792154106638e+17, 3.736110301707989e+17, 6.324875022200906e+17] ["HPImt", null, "uoCqzWAZbfyPT", "gjAycoPu", null] desktops bigint_col +1 77 238055689 703523995378749126 true 5.8739328E8 \N Obesity significant seemingly december. Mutual handling brush. Respective outlet apparent carriage condition. Deliver rarely message magazine reward. quotation energy heighten 2022-08-27T03:36:31 985381.2251 desktops bigint_col \N [] ["rDSkRmOZxFm", "rCiLqodvJUnZGuCl"] desktops bigint_col +1 83 797661957 837249969127716283 false 2.6391996E7 3.054802436133497E17 Prevalence meaning spell brilliant bonus control dispute. Harm spark lie1 example tradition physics cheer. Funding high store halfway stimulus exhibit. Cute costume mysterious contempt mainstream. Buck kind barrel local garden. ought sporting 2022-08-16T23:29:46 310008.9678 desktops bigint_col 2020-06-30 [6.145991991429664e+17] ["bxhXIyxPDLd", "jKspKbihIfcg", "hyi", "RiYudALJAzFBAM", "sCUkfo", "XDQE", "alzgvO"] desktops bigint_col +1 85 741388831 956390649934952628 \N 7.6506342E8 7.8871090083722931E17 Bureaucracy accordingly landscape stark predict. Odds perspective cater miner coloured. Torture riot expire schedule. Sacrifice unprecedented denounce round. biology idiot 2022-08-08T04:26:20 \N desktops bigint_col \N [6.901511195860168e+17, 3.859761682785134e+16, 6.169197677431471e+17, 3.125001437485673e+17, 4.467898541676541e+16] [null, "yTfkbpmIecYwPF", null, "NFxcyq", "jhIslvprFgnYc"] desktops bigint_col +2 \N 512348397 556071212294545151 true 8.6100269E8 9.4859583110848256E17 Pledge trainer hang visit. Bye dad intake. Aspire speculate even highlight mayor portray. Cottage tune march. Spell formulate religious reside. tape lifestyle coverage 2022-08-21T10:30:33 469496.0119 desktops bigint_col 2021-08-08 [8.834857266908703e+17, 9.773573805703795e+16, 1.5345766649441316e+16] ["kVToBxzgeDlDCD", "QqKB", "OXEsmVBfWhKQPywUXj", null, "chffMKypYJZDQD", "kLyKQbchdAccV"] desktops bigint_col +2 \N 605386450 161034749716124783 true 5.3331568E8 8.483662331540023E17 Framework define container tomorrow disruption uniform soccer. Chocolate entry robust constitutional heating. Silk temporarily claim survivor flavour grant rental. Nobody carefully coffee adverse initial choice cream. Install diversity drown throughout magazine importance. lucky november 2022-08-10T10:45:05 915079.5564 desktops bigint_col \N [3.4137894979448314e+17, 5.6916409629776275e+17, 9.175450825189115e+17] ["YNUoOphUUFih", "anEahlvkTFWoemt"] desktops bigint_col +2 12 597868450 970898867721721145 \N 7.2001677E8 \N Go bank butter surprised broad mainstream photographer. Extremely noble fabulous tall. Range clean library poet. Cream prince terrorism. Threat applicant well worst hopefully. problem supervise vital theory 2022-08-30T21:48:47 968233.9679 desktops bigint_col 2017-09-10 [3.2150164349149677e+17, 5.6837981064709496e+16, 1.1855546027353826e+17] ["ExUpe"] desktops bigint_col +2 12 778351598 178745089173303799 \N 1.87834704E8 4.0492247739101331E17 Stark readily word precedent trillion yesterday. Terrain skill harassment supporter bank competence. Critically enthusiasm panic. compensation pleased bury enter figure 2022-08-09T01:15:27 862233.3268 desktops bigint_col 2015-11-12 [4.059834315213429e+16, 9.949490374303972e+17] ["LpCxUwhIAHb", "OlxTdIU", null, "MTKAIFaSjnlwlsZ"] desktops bigint_col +2 13 284375569 313022208823056690 true 8.4682362E8 1.80727977302818752E17 West appealing heavy workout next. Security gentle chairman control indicator. deploy vice two programme 2022-08-08T13:15:19 499389.8483 desktops bigint_col \N [2.6052619610612326e+17, 1.6511987897783744e+16] [] desktops bigint_col +2 25 400500070 943655591782324457 true 4.70849024E8 8.589732873557929E17 Mercy welfare snap definition paint laser. System revive stumble highlight balanced gaming bombing. artificial appointment trigger horizon win \N 800687.0791 desktops bigint_col 2013-01-25 [6.036467426448096e+17, 2.1130489046972413e+17] ["hrayImCOtXikHqCGfBih"] desktops bigint_col +2 28 515357905 487459024969808193 true 2.2465104E8 9.8016470740501632E17 Must ease comprise drain specialize frighten wine. Mum pronounced quickly resist fail. Even stall marker. Not according constitution judgement. Explicit elaborate supervision ski feeding plane. heating prescribe shaped end 2022-09-03T07:01:54 953756.2917 desktops bigint_col 2015-05-15 [] ["JWybUEoqerKOEvTGMhvc", "JysdjBlWQKLcvIglA", "XNeuEIVeYzRpy"] desktops bigint_col +2 29 726080366 398808167349063190 false 3.36853632E8 7.8843413319848051E17 Wholly phone succession coat criticism. Analyse overlook leg violation home vessel. Further careless coast. Mutual cost exam correspondence mail despite. unconscious rating 2022-08-13T14:26:23 \N desktops bigint_col 2014-05-17 [2.2106660943234602e+17, 3.6934195934907565e+17, 8.861349520455558e+17, 6.523538011424809e+17, 3.788881047669972e+17] [] desktops bigint_col +2 32 227944144 699950355366195722 true 4.9220144E8 2.56063632494003872E17 Favour ray funeral accountable comment disaster. Warming fancy interactive signature. solo cycle 2022-08-15T08:47:53 288314.6424 desktops bigint_col 2017-12-22 [1.6780404003026906e+17, 9.205489892961844e+17] ["kTxHOdfJrwSi", "WthIZIQSgYFf", "cdMZPCeIjXIkgNOqkA", "pFLNZyW", "uOSVSHhrB", "SYgmtTJZLMzTMMDbauOc"] desktops bigint_col +2 33 425838871 943495121042472021 true 6.5801453E8 8.0715585698720205E17 Within bury ground span artificial popularity brother. Destroy successive meal attendance ban leg newspaper. Bend clinic especially. \N 2022-08-30T03:31:04 255834.5576 desktops bigint_col 2015-01-31 [9.816669775583758e+17, 8.183866158337096e+17, 3.614964995764407e+17, 6.51351375624206e+17] ["toKBkrCbdOZbesb", "eZT"] desktops bigint_col +2 33 621411734 223784268032178682 false 9.9321664E8 1.14314709029271536E17 \N feminist neither 2022-09-04T23:04 883603.3353 desktops bigint_col 2021-09-14 [6.219857054964748e+17, 7.51919542330081e+17, 4.981306541043197e+17] ["vSmAJAygtHrxHvYO", null, null] desktops bigint_col +2 43 8888148 838033351460227375 false 2.53972448E8 \N Rifle headquarters infect. Assistant below texture yet fork various. outlet mad worst enthusiasm 2022-08-19T01:19:19 105639.1166 desktops bigint_col 2013-09-07 [4.986002691572474e+17, 1.2809228753062507e+17, 5.0491976758581024e+17, 1.5552598850632848e+17] ["kWlBbSGSrQBQflrWAm", "mPnbBog", "kPdLH", "ymeCYO", "tqsTaa", "tnrangtfjQRU", "LTSzBEcbxwwNB", null] desktops bigint_col +2 44 191401327 671963452353742152 false 1.3778736E8 1.83243227484724992E17 Obey dead profitable. Interpretation anniversary joy neighbouring edit electric tidy. Copyright preservation stark deadly amid reminder mobilize. Among choir fork bored. Mortgage disturb long-time professional tongue. mathematics again \N 360637.6705 desktops bigint_col 2019-10-03 [7.77089975109847e+16] [] desktops bigint_col +2 46 650434655 39848157600020446 true 6.9146586E8 3.7060518421467648E17 Communicate club specialized recession kick legend. Stake insult early fluid ocean integration compound. Associate economics psychiatric beneath relaxed. several trait homeless pose 2022-08-10T23:20:59 403069.5119 desktops bigint_col 2020-03-09 [1.2093856458728192e+17, 1.3086043347585107e+17, 1.803255161213063e+17, 7.679491592398975e+17, 4.931067289222262e+17] ["FaCQGnfOPaUFEUrPVVpw", null] desktops bigint_col +2 50 760146996 480450791207697069 false 255391.28 3.9309384934358093E17 Broadcaster trailer twice diagram mountain phase. Of conference carpet invisible. endorse function 2022-08-21T01:11:11 636137.6081 desktops bigint_col \N [1.8884955224866605e+17, 5.598730802774882e+17, 4.158846805828165e+16] ["guIgqkPI", "onBpJgKTcUDh", "uESAiBGXJVA", "bLVptq"] desktops bigint_col +2 51 724892870 772630919179671999 true 7.2187706E8 2.26274676625050752E17 Sponsorship annoyed awful. Forth twist flight. Super stereotype nurse central. Trio dominance suspicious mechanic commentator warning. \N 2022-09-03T06:49:51 316122.7734 desktops bigint_col 2017-02-12 [] ["XRyNrLG", "mEbfBWAWcUcp", "KTvC", null, null, "BLLXRBys", "IFvSWbvmGQuIko", "BnlaBuIRcpXhrPqpV"] desktops bigint_col +2 62 947714244 451589400328043377 true 3.4991712E8 6.5850654142658534E17 Important dog constantly capable architect. Strongly strictly industrial creature temporary sufficient people. June connection fabulous. Motivation curriculum offence file tear1 true. July perception arena innovative. verse comprehensive war kingdom 2022-08-29T20:42:46 \N desktops bigint_col 2013-03-23 [8.553412772762889e+17, 5.227546601096389e+16, 6.754153329941062e+17] \N desktops bigint_col +2 66 \N 362061181042516667 \N 9.2758816E8 1.474065095112208E17 Boost tax identify organized processor wear. Than space identify. And scene horse. sale quota gun closed faith 2022-08-13T12:18:06 668437.3033 desktops bigint_col 2014-07-06 [9.125392280363515e+17, 5.451215191556551e+17, 2.877125037775009e+17, 1.2170082291089424e+17, 2.8477499789790096e+17] ["SKQpnBUqkgx", "hLuKeuoj", null, "kWdBDQbLMvhC", "qfZlNdoSPtewRM", "RcUHf", null, "wBRdsnuXf", "YTMUUkeblcUwHAS", "SffrfiBYbMhkICHSEkF"] desktops bigint_col +2 75 691337321 868250294959268172 true 7.8525376E8 \N Bedroom acquisition irrelevant cope parking charm call. Bright freeze corporation. Opposed net foreigner train. Infect seriously cycle expect. \N 2022-08-18T09:57:10 338665.2099 desktops bigint_col 2021-03-20 [7.231248276676792e+17, 5.276079075450041e+17, 6.5753225426992024e+16] ["ChM", "jzvpHVkhKJVM", null, "HabybUuFiIUxYQOmN", "axhqXUxkxFqiOojcq"] desktops bigint_col +2 81 \N 388342099454962780 false 9.1951021E8 3.1236464741799296E17 Towel disadvantage reveal major deficit ethic. Notebook please pay. Fixed bound push downtown contract. Memorial just currency humble pad gather rate. Installation dilemma freedom structural choir album. ensue courtesy 2022-09-01T21:50:30 186208.2738 desktops bigint_col 2018-12-01 [9.063784429530822e+17, 6.977872624277775e+17, 7.721732368138504e+17] ["ujeLWtyDmcBNnfiBEhyx", "lbCLmSFUh", "TjrDQcJtXJoBaK"] desktops bigint_col +2 83 12394764 675592792500868694 true 3.2378528E8 7.7310517741791206E17 Cotton aspire stick. Theirs brilliant crystal. action withdraw 2022-08-19T16:37 604330.4268 desktops bigint_col 2014-06-03 [9.524860908721682e+17, 3.4250654785536506e+17, 5.135658031439463e+17, 4.8666548104110784e+17] ["VeFWyGrnoNn", "NRONgxhotn", null, null, "ZWYKWoXhRyPnNcWJeYJu", null] desktops bigint_col +2 87 369321831 260373492213221036 true 1930301.2 6.9965213797235238E17 Nursery veteran seat sixteen feminist basically. Obvious successive marginal laptop aggressive remains meaning. T-shirt passion advance native current utterly. Republic soup vanish likewise motivation awkward. Three gift satisfy once short. political stumble theme allocate 2022-08-11T16:35:33 890894.1055 desktops bigint_col 2019-12-04 [9.821996086294243e+17] ["kKJINHjiotL", "ThdbODAnnUIpZli", "PkoolKnpBtZWyjt", "bvfNYKvuoyQmJATvBFNN", "NPxPf", "kPOpaqrey", "CkZ", "rhIfNqyVZlpBBwxMJOQ", "yijAtQDlNX", "bZZyrkbRjFqeoTyq"] desktops bigint_col +2 87 49257151 652600171802393243 true 7.5004333E8 3.1997831048427386E17 Ship tone section indoors successor fare deadline. Prisoner breed clothes. Collect popular price seriously eligible. Equipment obvious manage discretion addition. Situated acre sample convey priest shop. effectively foreigner material 2022-08-25T19:33:10 658743.1470 desktops bigint_col 2020-04-09 [1.3796744856031629e+17] [null, null, "xCIEDCLIIB", "oMrXxHBPgjxSYsRQMSDe", "nUptFJynf"] desktops bigint_col +2 87 815222915 439317556995246224 true \N \N Laughter soccer survival attain receipt. Start body hire stake. \N 2022-08-08T09:41:33 375983.1712 desktops bigint_col \N [1.7520331452734906e+17, 8.218320637011135e+17, 5.102686480109678e+17, 5.3854930022541715e+17] ["wJOcTZvWYVmdKAI", "OLfIAFJsmKk", "onrRnpBxuDJOyb", "HvGOLYbSWSioTFpJ", "gQgHurYxvybeC", "bTkonbxQEh"] desktops bigint_col +2 90 \N 81538723103026803 false 9.8741286E8 7.5386138519914906E17 Massive course height. Breath total physician. Holiday thank organization translation log. Figure transit abstract. Query executive institute pill equipment definite legislative. vibrant physical green complement altogether \N 496008.6909 desktops bigint_col 2021-08-15 [1.1563862484685605e+17, 2.208686748728399e+17, 3.4907002307698164e+16, 2.4108551188340698e+17, 9.985507838875311e+17] \N desktops bigint_col +2 90 174219368 674822431440767844 false 6.5953984E8 5.021006591370041E17 Backdrop participation great treat subsidy delicious council. Inability garden heat join technological assault petition. racial tide big infect 2022-08-08T23:15:15 341624.6584 desktops bigint_col 2018-10-04 [7.271072731686944e+17, 8.850691215139868e+17, 4.8463435466539315e+17, 7.25837216316081e+17] [] desktops bigint_col +2 98 \N 435601208177882399 false 1.1480215E7 7.2935720946187264E17 Enrol assessment tactical. Tongue online combine between. Temporary representative lie2 treat. Consecutive genuine fierce. consent distribution 2022-08-20T08:38:16 822616.1952 desktops bigint_col 2019-07-08 [1.5952729308323066e+17, 7.704485945731784e+17, 9.562956777912644e+17] ["LXOPWnyYizgmvj", "DTNIEBgcpjqDMGd", "OffmIClBBYi", "fMFpVfjclRTjx", "IJtTsiNJ", "TNqlGaPqnszbWpFmBxmL"] desktops bigint_col +2 98 188402749 1757547065004506 true 8.2675984E7 4.7484333110575418E17 Bishop robot capability prosecution level generally feeding. Scratch each drink. Duo major address escape. Sport shooting standing word robbery. Care rescue reduction productive. swear paragraph 2022-08-08T13:45 290677.1033 desktops bigint_col \N [7.46266874313248e+17] ["CnlIyCMVmigoDrhTuOp", "KvmGQHtpEiYDSXqMz", "KgCoET", "zOas", "SZPuCYbHQWeRcxmuynP", "mjdWpzTQcQy", "JWOXDHpZXmvjxL", "vQhkWMkDphjvCy"] desktops bigint_col +3 \N 783387927 833784751127412839 false 6.1395981E8 7.2964677883635776E17 \N soup voluntary ridiculous atmosphere liberal 2022-08-26T03:45:45 505553.1253 desktops bigint_col \N [8.846038446769828e+17, 8.355716311748035e+17, 2.5798603786894893e+17, 1.6821654273579677e+17, 9.633949134404147e+17] [null, "XGlJHIwttdCzqIVfWCmH", "XRvnqGd", "dLohrer", null] desktops bigint_col +3 11 452267632 180732776652068136 true 8.234185E8 5.0673167295151014E17 Subject pop reasoning annual release. Voluntary frightened counsellor edge demonstration congregation ashamed. Permanent exist dismiss. Laptop occasional casualty hundred. role tradition fluid hypothesis 2022-08-25T22:58:22 944476.9648 desktops bigint_col 2022-06-11 [] ["psilFGentDltGyjJ", "XhtWxdxoHdbG", "ahNRjrflBsPI", "cAubMaYWW", "KcpzcsaBgnyLOrChpSy", "ckBfKZeanAzCg"] desktops bigint_col +3 26 327351446 453711707916208854 false 1.3737296E8 5.3344294506759917E17 Legislation immune environment benefit. Blessing talent recovery. Heating cartoon lucky overlook texture noisy squad. Loom finger dealer circulate. snow fault 2022-08-27T19:27:19 577975.6390 desktops bigint_col \N [] ["SLPbAJYqM", "ImDnN"] desktops bigint_col +3 29 556030687 131420083813462395 true 6.3668006E8 1.68308855103597504E17 \N \N 2022-08-30T09:24:47 883705.3697 desktops bigint_col 2017-11-11 [6.742358467833588e+17, 7.746370686491464e+17] ["fZIwBBispsCxyXmN", null, "lJhklzHS", "evvakkUnLs", "jLiRDsCOdTpJrimosUfM"] desktops bigint_col +3 31 545321127 849437036516380697 false 6.7927539E8 8.889170476513504E17 Extraordinary graphic modern commissioner you treaty. Extensive humble carve. dot integrity friendly shell 2022-08-08T13:54:08 470633.1523 desktops bigint_col 2018-01-19 [8.45513368465698e+17, 2.138306506876291e+17, 6.729663167160476e+17, 1.1214072770538208e+17] ["HKHOIHUvShdJJSgr", "TncPINpiMUcG", "NvraQjiGStRHkWiZuyy", null, "bpDUJCQuTKlpyvQDJl", "xhkFefWnbE", "yifisHJugRi", "NmM", "ansFopZVwPKESg", "UqMiLmJjuBqEFGJaMsFu"] desktops bigint_col +3 31 637297379 681740620949095717 false 3.72037984E8 1.5311977505834896E17 Universal bureaucracy hello critically. Notorious questionnaire convincing thoroughly impact machine important. Rest practice tackle collapse. Steam limited cognitive. \N 2022-08-18T00:10:36 978275.4508 desktops bigint_col 2015-11-01 [6.730618515485289e+17, 7.211752076302418e+17] ["pXjW", "GZx", "LpfBOpdkIIDtzvwgbH", "XQnIEJQasHJuMEkDQ"] desktops bigint_col +3 33 416890308 864968854877102244 true 3.57917696E8 \N Wedding direct nevertheless tobacco advertisement adopt. Link tell plot. loan settlement bind cluster specialized 2022-08-17T18:36:23 757206.2270 desktops bigint_col 2020-11-29 [2.7678152423529523e+17] ["rDipxjS", "eWEYwMfsjQjHUSUr", "SziEffzHJLcLSpsPvYXT", "sZSztAbVxwRkyrEYDEHT", "sjRQSTgFPRtFmuTbwA", "FUffvrlx", "xotJqXwb", "KEEFKcViaiWSbYPMp"] desktops bigint_col +3 38 568808594 82321681584796394 true 4.78516768E8 2.43156758463730528E17 Centre noise patent tribunal neutral frequent. Occurrence judicial world art bounce. precision resign 2022-08-29T15:24:31 883708.1258 desktops bigint_col 2016-09-19 [3.080566843096494e+17, 7.818941860140127e+17, 3.048632989978904e+17, 1.2902961399744917e+17, 5.5187143156873e+17] [] desktops bigint_col +3 52 909991391 827842515928592069 true 1.18878936E8 6.7175271888105011E17 Helpful casino stair relevance wealth smash tape. Spring alert reduction merge lead1. Morning second1 description imprison language combination. Report capital unify normally rare. bonus confine prosecutor be dense \N 377193.7087 desktops bigint_col 2013-09-09 [] ["RRMxHGqkj", "fuFWB", "GbcPzzfpCQiBdTJfopKm", null] desktops bigint_col +3 57 482875653 526266436633468708 false 4.61775136E8 3.2053274668533485E17 Interface convince breakfast sufficiently dilemma. Rank burden boss press. Busy legend sing option. Objective analysis rental skiing transportation mobile get. Print whip unacceptable float spectrum extensive. doctor handy everyone mobilize deny 2022-08-26T02:59:56 115518.8019 desktops bigint_col 2016-09-27 [1.3560043005574606e+17] \N desktops bigint_col +3 58 352921058 648712597438227479 \N 3.3095744E8 7.7999726469300518E17 Intelligent crude worthy therefore birthday. Productive impression ecological thick. Detailed hostage tourist argue passive congratulate. Onion human terrorism tomorrow body media altogether. Kingdom lottery threat ambulance old-fashioned. it cigarette committee \N 324831.2649 desktops bigint_col 2013-12-27 [7.666249082433514e+17] ["TVbWbFUf", "iZPWCWemhEtgxWfk", "NyYOyTdBxloWNiBTiyP", "nqPSQlMUqireilv", "EicBNzLPCPNPAG", "MMpkckNiTyFug"] desktops bigint_col +3 65 318259243 788429562626231382 true 2.53879552E8 5.2861201434808768E16 Visitor trend hydrogen. Privilege relationship actor entertaining exposure legendary. Congratulate online theoretical whereby. Nineteen counsellor reasonably stretch legend. position employer 2022-09-03T12:12:11 608134.9893 desktops bigint_col 2015-05-19 [5.593220113166346e+17] ["VarUrIsYc", "pGFGIcKER"] desktops bigint_col +3 77 \N 899510136394242816 true 1.05334736E8 8.0177537887940186E17 Weight goal presidency loss. Settlement unveil do1 company carbon wealth. preach collective booking 2022-08-28T05:25:05 673575.5591 desktops bigint_col 2015-02-04 [4.0265714573684e+16, 3.6374623436062803e+17, 9.360348023765316e+17] ["CVc", "QVvmaPaOSUySeNcVA", "NYWhCcvGZkzhEJlQKRsU", "QsUiaIpIpeILEKAKlsKy", null, "iDCqoRFpQIxWCPLx", "qMIcGN", "ObOJlI"] desktops bigint_col +3 78 321907112 1365849332566195 true 1.85100944E8 1.08461021524732256E17 Mistake reign storm slowly wound. Still minute1 soft worried amusing record employee. Sand cheer descend serve initial intervene. Institution bean excuse sustainable cancel tribal. Pen coloured contemporary genre predecessor entrance. commentator expense learning 2022-08-25T11:20:08 750498.2441 desktops bigint_col 2022-06-07 [5.511430917788297e+17, 8.424231668826157e+17, 5.0052505348145914e+17] ["ezgdxpC", "vgO", "OMkMCZeILnonBZ", "QyurcM", "xzTmLloMMHqeLDHcC", "lTBEXOjqFmwpCjDIiDOw", "Igu"] desktops bigint_col +3 78 393296774 707084360255744262 true 6.189552E8 \N Ghost layout teens lemon racing until. Every cheerful kitchen so pop near. Coup dull besides. Call society bath willingness rat upcoming sick. \N 2022-08-25T02:36:50 991688.3946 desktops bigint_col 2015-02-11 [9.391942870795297e+17, 6.644484220856004e+17, 7.609218522018826e+17, 7.100199403632896e+17, 1.0405543615581325e+17] ["zDCQKzRPThjzdqteoA", "zaVw", "ijdefQyGGW", "Sdq", "AqPQglMVu", null, "XtDbqkgyUyoYLYOgQt"] desktops bigint_col +3 96 230092499 696089533808779510 false 3.80778464E8 1.05022936741982704E17 Hilarious banana enthusiast show. Gathering edge robust drunk truly mortgage. Equation stun receipt. Client resolution november weird devise store. disappointment case 2022-08-08T19:38:23 129371.4518 desktops bigint_col 2014-12-28 [2.178241064662456e+17, 3.118728389707264e+17] \N desktops bigint_col +3 97 243052497 157769598533078600 true 9.4151667E8 6.872807770432976E16 Shake chop hospital leave. Nest part-time thankfully interested. Household shortage around kind pirate pride spending. consideration pastor 2022-08-13T06:38:15 110254.9084 desktops bigint_col 2015-01-19 [2.504371687444008e+17, 4.9783978018663245e+17] ["jev", "ddhryCfoZgWyNEGZ"] desktops bigint_col +3 99 778108157 61873080930551778 true 9.8627565E8 \N Technology gene shame stock grandfather initiate cluster. Sympathy including grind right. Colleague pour interference vanish eligible. Campaign fold poison various between. Formal proper prize ray do1. \N 2022-08-29T19:43:46 908171.0209 desktops bigint_col 2020-02-29 [8.289095771326624e+17, 1.9573475208108093e+17, 6.703456714526179e+17, 7.245677936816932e+17] ["zJldV", "BWhpggYvPlozXm", "bAbglLACbIPnJXpyjDIa"] desktops bigint_col + +-- !desc_2 -- +bigint_col BIGINT Yes false \N NONE +binary_col TEXT Yes false \N NONE +boolean_col BOOLEAN Yes false \N NONE +char_col CHAR(50) Yes false \N NONE +date_col DATE Yes false \N NONE +decimal_col DECIMAL(12, 4) Yes false \N NONE +double_col DOUBLE Yes false \N NONE +float_col FLOAT Yes false \N NONE +int_col INT Yes true \N +list_double_col ARRAY Yes false \N NONE +list_string_col ARRAY Yes false \N NONE +smallint_col SMALLINT Yes true \N +string_col TEXT Yes false \N NONE +timestamp_col DATETIME(6) Yes false \N NONE +tinyint_col TINYINT Yes true \N +varchar_col TEXT Yes false \N NONE + +-- !select_2 -- +1 \N \N 757403305318104467 false 3.26199968E8 1.0049111235672792E17 Consolidate iron breakfast inhibit obesity mount hearing. Limitation bite sibling creation between sound. Plus1 layer injury favourable detain. Learn pronounced entrepreneur personnel wool strive. Pose curiosity spite absolutely combination right. \N 2022-08-11T10:09:31 996888.8617 desktops bigint_col 2015-08-24 [5.084045411017597e+17, 3.942856911182207e+17, 8.38109720690003e+17, 5.0079271855467546e+17] ["NRcqedH", "JIkT", "JXw", "JLvj"] +1 \N 107386238 108422142707872166 true 8.3058278E8 3.7039169232624294E17 Cut god fade script breakfast delete. Reasoning enthusiast column genetic tolerate cop school. false merge onto 2022-08-18T16:59:24 549352.8096 desktops bigint_col 2015-10-23 [] ["URknu", "WpuINXTasndUKtb", "tSIMVWOtOQy", "nmFV", "QRnFVjUex"] +1 \N 653934097 947581034711672436 false 3.82426912E8 \N Sorry shortage prevalence dismissal probability warrior motorist. Discover cancer gate spell. Initially investigator opt cannot colourful production. Shoot capable concentration jail utilize. patent disastrous 2022-08-10T06:28:28 534581.4191 desktops bigint_col 2012-10-22 [9.955149870448799e+17, 5.458963138726759e+17] ["kThacn", "iqAYojfNU", "OmajtKJm", "qfWesYxCEJWqTS", "ZrwOMXcp", "DPeldCDbfKlJuGZKYwTK"] +1 11 809290623 686793101332632 true 4.71326496E8 3.3112444749135328E16 Dream biology decline modification councillor large-scale stair. Perfectly possess abundance. Constitution carbon amazed declare opposed lesser supervisor. Accountant furious consolidate atmosphere pin direction this. Creator illustrate yell viewer ankle. disclosure old appetite \N 231822.0509 desktops bigint_col 2022-02-23 [] \N +1 12 720480298 439676960593506542 true 8.7667955E8 \N Revision flying spelling steer vicious apparent. Tendency follow must adhere reply festival. half title stupid virtual 2022-08-26T05:05:27 644585.1705 desktops bigint_col \N [5.799499253822285e+17, 6.977374510334419e+17, 2.8792573287467094e+17] [] +1 12 872242197 39372924164800285 true 4.02162976E8 5.4119588683596512E17 Bulk fundamental disagree bench past acknowledge. Mercy toxic sing. might chef \N 388501.8604 desktops bigint_col 2022-08-29 [] [null, "hYetKKEDJm"] +1 14 726053880 161015669819134929 false 4.9883168E8 4.3323385209889466E17 Team always illusion advantage could. Instrumental stability seven manager care declare. Jewellery navigation bee creature violence bathroom or. Routine identity application presume him. Pan emerge temporarily. counter consciousness study 2022-08-08T13:52:40 366074.9783 desktops bigint_col 2013-02-09 [1.7713806707666048e+17, 3.7383671495565165e+17, 6.259419911350532e+17, 2.5287201471209952e+17] ["vaSfuLXpBIzKBLnaBd", "DXbG", "OaCJCGdl"] +1 15 98067009 107002720698930270 true 1.01148912E8 5.7735895489617459E17 Single misleading neutral unknown footage above kind. Walk insect harvest hair suspicion traffic. Slap endure text instrument naval again. \N 2022-08-13T01:39:09 332059.6117 desktops bigint_col 2013-04-09 [7.035787745720701e+17] ["wBJJPblSffQaeWC", "OwhqU", "EqotFomTShNObQcm", null, "nMRD", null, "yhtbbFbanWuSmjycpgLj", "ALyVKaolMG", null] +1 20 86450486 128585355387472229 true 3.85727264E8 8.0578324815203955E17 Renew review concentration child pronounce nod emergence. Due travel letter red. private each rocket hers 2022-08-11T04:59:32 644172.2438 desktops bigint_col 2017-12-20 [] [null, "BJWAHVFXmUmTgeJ", "yGZpeeuEAKC"] +1 20 907476179 51144570531813048 \N 1.14442112E8 6.3573879782317082E17 Guilt decide electronic expectation potential outdoors. Magnitude unable furious. healthy bike controversy 2022-09-03T09:08:57 368422.0700 desktops bigint_col 2021-06-18 [9.10700912260912e+17, 4.405327840725512e+17] [] +1 28 837555130 230916677115824233 true 5.4569421E8 2.70471322007323488E17 Cousin artificial abandon custody beam provision. Chemical abroad human. Arms young see. handy initial neat navigation 2022-09-03T17:55:12 524469.0578 desktops bigint_col 2018-09-04 [3.855014125096763e+17, 2.1039308993257866e+17, 9.31498606282679e+17] ["lgtZYBShnyC", "nRQjgF", "aCVIMQpZuYqOig", "lbVNNHeRK", "HsOO", "IJYFhh", "MDthVyeZpCjF", "sMnbIiEFr", "CyLO", "UAWGAZiQDzkpQBYD"] +1 28 838335879 382950133409826292 true 6.2795968E8 3.7588441507785466E17 Evaluate locate guitar frightened. Premier warrior organized buy eat. About die rock creep. civilization discretion desert connect firework 2022-09-02T22:27:57 479988.5725 desktops bigint_col 2020-12-16 [7.804613874622225e+17, 1.8854381373872086e+17] ["Mfwddzogb", "LGdzf", "gEQy"] +1 31 \N 741341588454293823 true 3.86187136E8 4.6610951470341754E17 Aspiration deployment tough make-up. Listing bride hole revision criticize ray sponsorship. Former whereby vanish strain typically. Transition lost erect settle provoke fate. Action atrocity wildlife weird helpful inclined. vary matching \N 389141.8253 desktops bigint_col 2019-04-30 [2.6031378443226595e+17, 9.016824926976108e+17, 5.316609472515438e+17, 3.669648424112748e+17] [null, "EEXcpNyLcwWMyfI"] +1 36 260311723 663342854057084267 false 7.9524992E8 \N Surround characterize wonder. Lady inner cost. Succession clerk increase religious governor squad. Reconstruction commission immediate bit hurt debate. variable contractor female pastor 2022-09-03T19:12:06 947575.7788 desktops bigint_col 2020-12-23 [] ["qiqjac", "SRZCFcqIelw", "rPDODktyX", "QYIZgxOhzLpMIKkzYik", "yfrsvMThiRjyFUypbZ", "WaVFzUSxW", "PRPQuClAUEdqbI", "WFA", "vbQoqNFLfaWRFVcrDX", "lfW"] +1 40 706858385 86990101748729070 true 5.4196077E8 6.9602303815701107E17 Resemble detection wood give launch distinction. Per attention deliberate mum. Modification segment foreigner complement conflict. Travel historic over advertisement. Baby recover soft satisfaction relatively. litter alone 2022-08-08T20:38:09 922369.7339 desktops bigint_col 2014-11-13 [5.85367882988869e+17, 6.078596726647018e+17] ["mDBdSwBdKdvhBaComWk", null, "rpVrUUrGMTtJ", "RYTzZRieSBWkDMiJF", "FapEEnbaHD", null, "Gxqzfggamjd", "DMd", "SFMJCivDFFiXOak", "reEcSR"] +1 56 563643370 \N false 7.2356896E8 8.2856413152776218E17 \N since super fare wipe 2022-08-14T17:47:58 105736.4432 desktops bigint_col 2012-10-25 [9.610180903194888e+17, 7.288131679313782e+17, 8.355045939461916e+17, 3.4987201052222304e+17] ["hoZwRj", "cMhAJCpHrUZQ", "ocYAxs", "gGVqSQewuauBCUkvzr", "yckCKXxVhrdJp", "eZCHD"] +1 58 968282995 675983799610076771 true 1.7376632E8 1.67326199513268896E17 \N \N 2022-08-11T17:23:24 136928.4456 desktops bigint_col 2016-03-24 [6.426187514709601e+17, 8.332563760183391e+17, 5.7197719695391334e+17] ["rpBteuiLfmGcIIqP", "vBeWoaqiXYckauaJ", "kUepRQOLvwqc", "kbIBPMSLEgpaACkdhI", "nBQuR", null, "RPFYYfTTHqI", "iflOfcPV", "qMIr"] +1 63 \N 122703346364718870 false 9.4316915E8 7.1076074324903078E17 Found illegal battery exam originally depressing profound. Too despite pot old go apparatus description. Goal consensus update take. beneficial lunch troop restrict 2022-08-26T10:00:47 339452.6832 desktops bigint_col 2017-10-07 [9.981882027272243e+17, 6.346875650940211e+17] \N +1 68 317879314 979790364239098905 true 2.34733152E8 9.8859784125654656E16 Procedure can2 dialogue. Why promise wing carefully functional. Enthusiast active chop reality eventually thirsty climb. Crystal rule nearby pop disagree picture. special phone immense \N 517206.9006 desktops bigint_col 2019-07-10 [] \N +1 73 \N 41268263353414053 true 4.40259392E8 \N Accumulate expand deadly danger scheme acute privatization. Touch county independence. Demand medication estate high-profile amid explosion. encompass icon end 2022-08-14T04:16:56 544035.6731 desktops bigint_col 2013-05-15 [3.104792154106638e+17, 3.736110301707989e+17, 6.324875022200906e+17] ["HPImt", null, "uoCqzWAZbfyPT", "gjAycoPu", null] +1 77 238055689 703523995378749126 true 5.8739328E8 \N Obesity significant seemingly december. Mutual handling brush. Respective outlet apparent carriage condition. Deliver rarely message magazine reward. quotation energy heighten 2022-08-27T03:36:31 985381.2251 desktops bigint_col \N [] ["rDSkRmOZxFm", "rCiLqodvJUnZGuCl"] +1 83 797661957 837249969127716283 false 2.6391996E7 3.054802436133497E17 Prevalence meaning spell brilliant bonus control dispute. Harm spark lie1 example tradition physics cheer. Funding high store halfway stimulus exhibit. Cute costume mysterious contempt mainstream. Buck kind barrel local garden. ought sporting 2022-08-16T23:29:46 310008.9678 desktops bigint_col 2020-06-30 [6.145991991429664e+17] ["bxhXIyxPDLd", "jKspKbihIfcg", "hyi", "RiYudALJAzFBAM", "sCUkfo", "XDQE", "alzgvO"] +1 85 741388831 956390649934952628 \N 7.6506342E8 7.8871090083722931E17 Bureaucracy accordingly landscape stark predict. Odds perspective cater miner coloured. Torture riot expire schedule. Sacrifice unprecedented denounce round. biology idiot 2022-08-08T04:26:20 \N desktops bigint_col \N [6.901511195860168e+17, 3.859761682785134e+16, 6.169197677431471e+17, 3.125001437485673e+17, 4.467898541676541e+16] [null, "yTfkbpmIecYwPF", null, "NFxcyq", "jhIslvprFgnYc"] +2 \N 512348397 556071212294545151 true 8.6100269E8 9.4859583110848256E17 Pledge trainer hang visit. Bye dad intake. Aspire speculate even highlight mayor portray. Cottage tune march. Spell formulate religious reside. tape lifestyle coverage 2022-08-21T10:30:33 469496.0119 desktops bigint_col 2021-08-08 [8.834857266908703e+17, 9.773573805703795e+16, 1.5345766649441316e+16] ["kVToBxzgeDlDCD", "QqKB", "OXEsmVBfWhKQPywUXj", null, "chffMKypYJZDQD", "kLyKQbchdAccV"] +2 \N 605386450 161034749716124783 true 5.3331568E8 8.483662331540023E17 Framework define container tomorrow disruption uniform soccer. Chocolate entry robust constitutional heating. Silk temporarily claim survivor flavour grant rental. Nobody carefully coffee adverse initial choice cream. Install diversity drown throughout magazine importance. lucky november 2022-08-10T10:45:05 915079.5564 desktops bigint_col \N [3.4137894979448314e+17, 5.6916409629776275e+17, 9.175450825189115e+17] ["YNUoOphUUFih", "anEahlvkTFWoemt"] +2 12 597868450 970898867721721145 \N 7.2001677E8 \N Go bank butter surprised broad mainstream photographer. Extremely noble fabulous tall. Range clean library poet. Cream prince terrorism. Threat applicant well worst hopefully. problem supervise vital theory 2022-08-30T21:48:47 968233.9679 desktops bigint_col 2017-09-10 [3.2150164349149677e+17, 5.6837981064709496e+16, 1.1855546027353826e+17] ["ExUpe"] +2 12 778351598 178745089173303799 \N 1.87834704E8 4.0492247739101331E17 Stark readily word precedent trillion yesterday. Terrain skill harassment supporter bank competence. Critically enthusiasm panic. compensation pleased bury enter figure 2022-08-09T01:15:27 862233.3268 desktops bigint_col 2015-11-12 [4.059834315213429e+16, 9.949490374303972e+17] ["LpCxUwhIAHb", "OlxTdIU", null, "MTKAIFaSjnlwlsZ"] +2 13 284375569 313022208823056690 true 8.4682362E8 1.80727977302818752E17 West appealing heavy workout next. Security gentle chairman control indicator. deploy vice two programme 2022-08-08T13:15:19 499389.8483 desktops bigint_col \N [2.6052619610612326e+17, 1.6511987897783744e+16] [] +2 25 400500070 943655591782324457 true 4.70849024E8 8.589732873557929E17 Mercy welfare snap definition paint laser. System revive stumble highlight balanced gaming bombing. artificial appointment trigger horizon win \N 800687.0791 desktops bigint_col 2013-01-25 [6.036467426448096e+17, 2.1130489046972413e+17] ["hrayImCOtXikHqCGfBih"] +2 28 515357905 487459024969808193 true 2.2465104E8 9.8016470740501632E17 Must ease comprise drain specialize frighten wine. Mum pronounced quickly resist fail. Even stall marker. Not according constitution judgement. Explicit elaborate supervision ski feeding plane. heating prescribe shaped end 2022-09-03T07:01:54 953756.2917 desktops bigint_col 2015-05-15 [] ["JWybUEoqerKOEvTGMhvc", "JysdjBlWQKLcvIglA", "XNeuEIVeYzRpy"] +2 29 726080366 398808167349063190 false 3.36853632E8 7.8843413319848051E17 Wholly phone succession coat criticism. Analyse overlook leg violation home vessel. Further careless coast. Mutual cost exam correspondence mail despite. unconscious rating 2022-08-13T14:26:23 \N desktops bigint_col 2014-05-17 [2.2106660943234602e+17, 3.6934195934907565e+17, 8.861349520455558e+17, 6.523538011424809e+17, 3.788881047669972e+17] [] +2 32 227944144 699950355366195722 true 4.9220144E8 2.56063632494003872E17 Favour ray funeral accountable comment disaster. Warming fancy interactive signature. solo cycle 2022-08-15T08:47:53 288314.6424 desktops bigint_col 2017-12-22 [1.6780404003026906e+17, 9.205489892961844e+17] ["kTxHOdfJrwSi", "WthIZIQSgYFf", "cdMZPCeIjXIkgNOqkA", "pFLNZyW", "uOSVSHhrB", "SYgmtTJZLMzTMMDbauOc"] +2 33 425838871 943495121042472021 true 6.5801453E8 8.0715585698720205E17 Within bury ground span artificial popularity brother. Destroy successive meal attendance ban leg newspaper. Bend clinic especially. \N 2022-08-30T03:31:04 255834.5576 desktops bigint_col 2015-01-31 [9.816669775583758e+17, 8.183866158337096e+17, 3.614964995764407e+17, 6.51351375624206e+17] ["toKBkrCbdOZbesb", "eZT"] +2 33 621411734 223784268032178682 false 9.9321664E8 1.14314709029271536E17 \N feminist neither 2022-09-04T23:04 883603.3353 desktops bigint_col 2021-09-14 [6.219857054964748e+17, 7.51919542330081e+17, 4.981306541043197e+17] ["vSmAJAygtHrxHvYO", null, null] +2 43 8888148 838033351460227375 false 2.53972448E8 \N Rifle headquarters infect. Assistant below texture yet fork various. outlet mad worst enthusiasm 2022-08-19T01:19:19 105639.1166 desktops bigint_col 2013-09-07 [4.986002691572474e+17, 1.2809228753062507e+17, 5.0491976758581024e+17, 1.5552598850632848e+17] ["kWlBbSGSrQBQflrWAm", "mPnbBog", "kPdLH", "ymeCYO", "tqsTaa", "tnrangtfjQRU", "LTSzBEcbxwwNB", null] +2 44 191401327 671963452353742152 false 1.3778736E8 1.83243227484724992E17 Obey dead profitable. Interpretation anniversary joy neighbouring edit electric tidy. Copyright preservation stark deadly amid reminder mobilize. Among choir fork bored. Mortgage disturb long-time professional tongue. mathematics again \N 360637.6705 desktops bigint_col 2019-10-03 [7.77089975109847e+16] [] +2 46 650434655 39848157600020446 true 6.9146586E8 3.7060518421467648E17 Communicate club specialized recession kick legend. Stake insult early fluid ocean integration compound. Associate economics psychiatric beneath relaxed. several trait homeless pose 2022-08-10T23:20:59 403069.5119 desktops bigint_col 2020-03-09 [1.2093856458728192e+17, 1.3086043347585107e+17, 1.803255161213063e+17, 7.679491592398975e+17, 4.931067289222262e+17] ["FaCQGnfOPaUFEUrPVVpw", null] +2 50 760146996 480450791207697069 false 255391.28 3.9309384934358093E17 Broadcaster trailer twice diagram mountain phase. Of conference carpet invisible. endorse function 2022-08-21T01:11:11 636137.6081 desktops bigint_col \N [1.8884955224866605e+17, 5.598730802774882e+17, 4.158846805828165e+16] ["guIgqkPI", "onBpJgKTcUDh", "uESAiBGXJVA", "bLVptq"] +2 51 724892870 772630919179671999 true 7.2187706E8 2.26274676625050752E17 Sponsorship annoyed awful. Forth twist flight. Super stereotype nurse central. Trio dominance suspicious mechanic commentator warning. \N 2022-09-03T06:49:51 316122.7734 desktops bigint_col 2017-02-12 [] ["XRyNrLG", "mEbfBWAWcUcp", "KTvC", null, null, "BLLXRBys", "IFvSWbvmGQuIko", "BnlaBuIRcpXhrPqpV"] +2 62 947714244 451589400328043377 true 3.4991712E8 6.5850654142658534E17 Important dog constantly capable architect. Strongly strictly industrial creature temporary sufficient people. June connection fabulous. Motivation curriculum offence file tear1 true. July perception arena innovative. verse comprehensive war kingdom 2022-08-29T20:42:46 \N desktops bigint_col 2013-03-23 [8.553412772762889e+17, 5.227546601096389e+16, 6.754153329941062e+17] \N +2 66 \N 362061181042516667 \N 9.2758816E8 1.474065095112208E17 Boost tax identify organized processor wear. Than space identify. And scene horse. sale quota gun closed faith 2022-08-13T12:18:06 668437.3033 desktops bigint_col 2014-07-06 [9.125392280363515e+17, 5.451215191556551e+17, 2.877125037775009e+17, 1.2170082291089424e+17, 2.8477499789790096e+17] ["SKQpnBUqkgx", "hLuKeuoj", null, "kWdBDQbLMvhC", "qfZlNdoSPtewRM", "RcUHf", null, "wBRdsnuXf", "YTMUUkeblcUwHAS", "SffrfiBYbMhkICHSEkF"] +2 75 691337321 868250294959268172 true 7.8525376E8 \N Bedroom acquisition irrelevant cope parking charm call. Bright freeze corporation. Opposed net foreigner train. Infect seriously cycle expect. \N 2022-08-18T09:57:10 338665.2099 desktops bigint_col 2021-03-20 [7.231248276676792e+17, 5.276079075450041e+17, 6.5753225426992024e+16] ["ChM", "jzvpHVkhKJVM", null, "HabybUuFiIUxYQOmN", "axhqXUxkxFqiOojcq"] +2 81 \N 388342099454962780 false 9.1951021E8 3.1236464741799296E17 Towel disadvantage reveal major deficit ethic. Notebook please pay. Fixed bound push downtown contract. Memorial just currency humble pad gather rate. Installation dilemma freedom structural choir album. ensue courtesy 2022-09-01T21:50:30 186208.2738 desktops bigint_col 2018-12-01 [9.063784429530822e+17, 6.977872624277775e+17, 7.721732368138504e+17] ["ujeLWtyDmcBNnfiBEhyx", "lbCLmSFUh", "TjrDQcJtXJoBaK"] +2 83 12394764 675592792500868694 true 3.2378528E8 7.7310517741791206E17 Cotton aspire stick. Theirs brilliant crystal. action withdraw 2022-08-19T16:37 604330.4268 desktops bigint_col 2014-06-03 [9.524860908721682e+17, 3.4250654785536506e+17, 5.135658031439463e+17, 4.8666548104110784e+17] ["VeFWyGrnoNn", "NRONgxhotn", null, null, "ZWYKWoXhRyPnNcWJeYJu", null] +2 87 369321831 260373492213221036 true 1930301.2 6.9965213797235238E17 Nursery veteran seat sixteen feminist basically. Obvious successive marginal laptop aggressive remains meaning. T-shirt passion advance native current utterly. Republic soup vanish likewise motivation awkward. Three gift satisfy once short. political stumble theme allocate 2022-08-11T16:35:33 890894.1055 desktops bigint_col 2019-12-04 [9.821996086294243e+17] ["kKJINHjiotL", "ThdbODAnnUIpZli", "PkoolKnpBtZWyjt", "bvfNYKvuoyQmJATvBFNN", "NPxPf", "kPOpaqrey", "CkZ", "rhIfNqyVZlpBBwxMJOQ", "yijAtQDlNX", "bZZyrkbRjFqeoTyq"] +2 87 49257151 652600171802393243 true 7.5004333E8 3.1997831048427386E17 Ship tone section indoors successor fare deadline. Prisoner breed clothes. Collect popular price seriously eligible. Equipment obvious manage discretion addition. Situated acre sample convey priest shop. effectively foreigner material 2022-08-25T19:33:10 658743.1470 desktops bigint_col 2020-04-09 [1.3796744856031629e+17] [null, null, "xCIEDCLIIB", "oMrXxHBPgjxSYsRQMSDe", "nUptFJynf"] +2 87 815222915 439317556995246224 true \N \N Laughter soccer survival attain receipt. Start body hire stake. \N 2022-08-08T09:41:33 375983.1712 desktops bigint_col \N [1.7520331452734906e+17, 8.218320637011135e+17, 5.102686480109678e+17, 5.3854930022541715e+17] ["wJOcTZvWYVmdKAI", "OLfIAFJsmKk", "onrRnpBxuDJOyb", "HvGOLYbSWSioTFpJ", "gQgHurYxvybeC", "bTkonbxQEh"] +2 90 \N 81538723103026803 false 9.8741286E8 7.5386138519914906E17 Massive course height. Breath total physician. Holiday thank organization translation log. Figure transit abstract. Query executive institute pill equipment definite legislative. vibrant physical green complement altogether \N 496008.6909 desktops bigint_col 2021-08-15 [1.1563862484685605e+17, 2.208686748728399e+17, 3.4907002307698164e+16, 2.4108551188340698e+17, 9.985507838875311e+17] \N +2 90 174219368 674822431440767844 false 6.5953984E8 5.021006591370041E17 Backdrop participation great treat subsidy delicious council. Inability garden heat join technological assault petition. racial tide big infect 2022-08-08T23:15:15 341624.6584 desktops bigint_col 2018-10-04 [7.271072731686944e+17, 8.850691215139868e+17, 4.8463435466539315e+17, 7.25837216316081e+17] [] +2 98 \N 435601208177882399 false 1.1480215E7 7.2935720946187264E17 Enrol assessment tactical. Tongue online combine between. Temporary representative lie2 treat. Consecutive genuine fierce. consent distribution 2022-08-20T08:38:16 822616.1952 desktops bigint_col 2019-07-08 [1.5952729308323066e+17, 7.704485945731784e+17, 9.562956777912644e+17] ["LXOPWnyYizgmvj", "DTNIEBgcpjqDMGd", "OffmIClBBYi", "fMFpVfjclRTjx", "IJtTsiNJ", "TNqlGaPqnszbWpFmBxmL"] +2 98 188402749 1757547065004506 true 8.2675984E7 4.7484333110575418E17 Bishop robot capability prosecution level generally feeding. Scratch each drink. Duo major address escape. Sport shooting standing word robbery. Care rescue reduction productive. swear paragraph 2022-08-08T13:45 290677.1033 desktops bigint_col \N [7.46266874313248e+17] ["CnlIyCMVmigoDrhTuOp", "KvmGQHtpEiYDSXqMz", "KgCoET", "zOas", "SZPuCYbHQWeRcxmuynP", "mjdWpzTQcQy", "JWOXDHpZXmvjxL", "vQhkWMkDphjvCy"] +3 \N 783387927 833784751127412839 false 6.1395981E8 7.2964677883635776E17 \N soup voluntary ridiculous atmosphere liberal 2022-08-26T03:45:45 505553.1253 desktops bigint_col \N [8.846038446769828e+17, 8.355716311748035e+17, 2.5798603786894893e+17, 1.6821654273579677e+17, 9.633949134404147e+17] [null, "XGlJHIwttdCzqIVfWCmH", "XRvnqGd", "dLohrer", null] +3 11 452267632 180732776652068136 true 8.234185E8 5.0673167295151014E17 Subject pop reasoning annual release. Voluntary frightened counsellor edge demonstration congregation ashamed. Permanent exist dismiss. Laptop occasional casualty hundred. role tradition fluid hypothesis 2022-08-25T22:58:22 944476.9648 desktops bigint_col 2022-06-11 [] ["psilFGentDltGyjJ", "XhtWxdxoHdbG", "ahNRjrflBsPI", "cAubMaYWW", "KcpzcsaBgnyLOrChpSy", "ckBfKZeanAzCg"] +3 26 327351446 453711707916208854 false 1.3737296E8 5.3344294506759917E17 Legislation immune environment benefit. Blessing talent recovery. Heating cartoon lucky overlook texture noisy squad. Loom finger dealer circulate. snow fault 2022-08-27T19:27:19 577975.6390 desktops bigint_col \N [] ["SLPbAJYqM", "ImDnN"] +3 29 556030687 131420083813462395 true 6.3668006E8 1.68308855103597504E17 \N \N 2022-08-30T09:24:47 883705.3697 desktops bigint_col 2017-11-11 [6.742358467833588e+17, 7.746370686491464e+17] ["fZIwBBispsCxyXmN", null, "lJhklzHS", "evvakkUnLs", "jLiRDsCOdTpJrimosUfM"] +3 31 545321127 849437036516380697 false 6.7927539E8 8.889170476513504E17 Extraordinary graphic modern commissioner you treaty. Extensive humble carve. dot integrity friendly shell 2022-08-08T13:54:08 470633.1523 desktops bigint_col 2018-01-19 [8.45513368465698e+17, 2.138306506876291e+17, 6.729663167160476e+17, 1.1214072770538208e+17] ["HKHOIHUvShdJJSgr", "TncPINpiMUcG", "NvraQjiGStRHkWiZuyy", null, "bpDUJCQuTKlpyvQDJl", "xhkFefWnbE", "yifisHJugRi", "NmM", "ansFopZVwPKESg", "UqMiLmJjuBqEFGJaMsFu"] +3 31 637297379 681740620949095717 false 3.72037984E8 1.5311977505834896E17 Universal bureaucracy hello critically. Notorious questionnaire convincing thoroughly impact machine important. Rest practice tackle collapse. Steam limited cognitive. \N 2022-08-18T00:10:36 978275.4508 desktops bigint_col 2015-11-01 [6.730618515485289e+17, 7.211752076302418e+17] ["pXjW", "GZx", "LpfBOpdkIIDtzvwgbH", "XQnIEJQasHJuMEkDQ"] +3 33 416890308 864968854877102244 true 3.57917696E8 \N Wedding direct nevertheless tobacco advertisement adopt. Link tell plot. loan settlement bind cluster specialized 2022-08-17T18:36:23 757206.2270 desktops bigint_col 2020-11-29 [2.7678152423529523e+17] ["rDipxjS", "eWEYwMfsjQjHUSUr", "SziEffzHJLcLSpsPvYXT", "sZSztAbVxwRkyrEYDEHT", "sjRQSTgFPRtFmuTbwA", "FUffvrlx", "xotJqXwb", "KEEFKcViaiWSbYPMp"] +3 38 568808594 82321681584796394 true 4.78516768E8 2.43156758463730528E17 Centre noise patent tribunal neutral frequent. Occurrence judicial world art bounce. precision resign 2022-08-29T15:24:31 883708.1258 desktops bigint_col 2016-09-19 [3.080566843096494e+17, 7.818941860140127e+17, 3.048632989978904e+17, 1.2902961399744917e+17, 5.5187143156873e+17] [] +3 52 909991391 827842515928592069 true 1.18878936E8 6.7175271888105011E17 Helpful casino stair relevance wealth smash tape. Spring alert reduction merge lead1. Morning second1 description imprison language combination. Report capital unify normally rare. bonus confine prosecutor be dense \N 377193.7087 desktops bigint_col 2013-09-09 [] ["RRMxHGqkj", "fuFWB", "GbcPzzfpCQiBdTJfopKm", null] +3 57 482875653 526266436633468708 false 4.61775136E8 3.2053274668533485E17 Interface convince breakfast sufficiently dilemma. Rank burden boss press. Busy legend sing option. Objective analysis rental skiing transportation mobile get. Print whip unacceptable float spectrum extensive. doctor handy everyone mobilize deny 2022-08-26T02:59:56 115518.8019 desktops bigint_col 2016-09-27 [1.3560043005574606e+17] \N +3 58 352921058 648712597438227479 \N 3.3095744E8 7.7999726469300518E17 Intelligent crude worthy therefore birthday. Productive impression ecological thick. Detailed hostage tourist argue passive congratulate. Onion human terrorism tomorrow body media altogether. Kingdom lottery threat ambulance old-fashioned. it cigarette committee \N 324831.2649 desktops bigint_col 2013-12-27 [7.666249082433514e+17] ["TVbWbFUf", "iZPWCWemhEtgxWfk", "NyYOyTdBxloWNiBTiyP", "nqPSQlMUqireilv", "EicBNzLPCPNPAG", "MMpkckNiTyFug"] +3 65 318259243 788429562626231382 true 2.53879552E8 5.2861201434808768E16 Visitor trend hydrogen. Privilege relationship actor entertaining exposure legendary. Congratulate online theoretical whereby. Nineteen counsellor reasonably stretch legend. position employer 2022-09-03T12:12:11 608134.9893 desktops bigint_col 2015-05-19 [5.593220113166346e+17] ["VarUrIsYc", "pGFGIcKER"] +3 77 \N 899510136394242816 true 1.05334736E8 8.0177537887940186E17 Weight goal presidency loss. Settlement unveil do1 company carbon wealth. preach collective booking 2022-08-28T05:25:05 673575.5591 desktops bigint_col 2015-02-04 [4.0265714573684e+16, 3.6374623436062803e+17, 9.360348023765316e+17] ["CVc", "QVvmaPaOSUySeNcVA", "NYWhCcvGZkzhEJlQKRsU", "QsUiaIpIpeILEKAKlsKy", null, "iDCqoRFpQIxWCPLx", "qMIcGN", "ObOJlI"] +3 78 321907112 1365849332566195 true 1.85100944E8 1.08461021524732256E17 Mistake reign storm slowly wound. Still minute1 soft worried amusing record employee. Sand cheer descend serve initial intervene. Institution bean excuse sustainable cancel tribal. Pen coloured contemporary genre predecessor entrance. commentator expense learning 2022-08-25T11:20:08 750498.2441 desktops bigint_col 2022-06-07 [5.511430917788297e+17, 8.424231668826157e+17, 5.0052505348145914e+17] ["ezgdxpC", "vgO", "OMkMCZeILnonBZ", "QyurcM", "xzTmLloMMHqeLDHcC", "lTBEXOjqFmwpCjDIiDOw", "Igu"] +3 78 393296774 707084360255744262 true 6.189552E8 \N Ghost layout teens lemon racing until. Every cheerful kitchen so pop near. Coup dull besides. Call society bath willingness rat upcoming sick. \N 2022-08-25T02:36:50 991688.3946 desktops bigint_col 2015-02-11 [9.391942870795297e+17, 6.644484220856004e+17, 7.609218522018826e+17, 7.100199403632896e+17, 1.0405543615581325e+17] ["zDCQKzRPThjzdqteoA", "zaVw", "ijdefQyGGW", "Sdq", "AqPQglMVu", null, "XtDbqkgyUyoYLYOgQt"] +3 96 230092499 696089533808779510 false 3.80778464E8 1.05022936741982704E17 Hilarious banana enthusiast show. Gathering edge robust drunk truly mortgage. Equation stun receipt. Client resolution november weird devise store. disappointment case 2022-08-08T19:38:23 129371.4518 desktops bigint_col 2014-12-28 [2.178241064662456e+17, 3.118728389707264e+17] \N +3 97 243052497 157769598533078600 true 9.4151667E8 6.872807770432976E16 Shake chop hospital leave. Nest part-time thankfully interested. Household shortage around kind pirate pride spending. consideration pastor 2022-08-13T06:38:15 110254.9084 desktops bigint_col 2015-01-19 [2.504371687444008e+17, 4.9783978018663245e+17] ["jev", "ddhryCfoZgWyNEGZ"] +3 99 778108157 61873080930551778 true 9.8627565E8 \N Technology gene shame stock grandfather initiate cluster. Sympathy including grind right. Colleague pour interference vanish eligible. Campaign fold poison various between. Formal proper prize ray do1. \N 2022-08-29T19:43:46 908171.0209 desktops bigint_col 2020-02-29 [8.289095771326624e+17, 1.9573475208108093e+17, 6.703456714526179e+17, 7.245677936816932e+17] ["zJldV", "BWhpggYvPlozXm", "bAbglLACbIPnJXpyjDIa"] + +-- !desc_3 -- +bigint_col BIGINT Yes false \N NONE +binary_col TEXT Yes false \N NONE +boolean_col BOOLEAN Yes false \N NONE +char_col TEXT Yes false \N NONE +date_col DATE Yes false \N NONE +decimal_col DECIMAL(12, 4) Yes false \N NONE +double_col DOUBLE Yes false \N NONE +float_col FLOAT Yes false \N NONE +int_col INT Yes true \N +list_double_col ARRAY Yes false \N NONE +list_string_col ARRAY Yes false \N NONE +smallint_col SMALLINT Yes true \N +string_col TEXT Yes false \N NONE +timestamp_col DATETIME(6) Yes false \N NONE +tinyint_col TINYINT Yes true \N +varchar_col TEXT Yes false \N NONE + +-- !select_3 -- +1 \N \N 757403305318104467 false 3.26199968E8 1.0049111235672792E17 Consolidate iron breakfast inhibit obesity mount hearing. Limitation bite sibling creation between sound. Plus1 layer injury favourable detain. Learn pronounced entrepreneur personnel wool strive. Pose curiosity spite absolutely combination right. \N 2022-08-11T10:09:31 996888.8617 desktops bigint_col 2015-08-24 [5.084045411017597e+17, 3.942856911182207e+17, 8.38109720690003e+17, 5.0079271855467546e+17] ["NRcqedH", "JIkT", "JXw", "JLvj"] +1 \N 107386238 108422142707872166 true 8.3058278E8 3.7039169232624294E17 Cut god fade script breakfast delete. Reasoning enthusiast column genetic tolerate cop school. false merge onto 2022-08-18T16:59:24 549352.8096 desktops bigint_col 2015-10-23 [] ["URknu", "WpuINXTasndUKtb", "tSIMVWOtOQy", "nmFV", "QRnFVjUex"] +1 \N 653934097 947581034711672436 false 3.82426912E8 \N Sorry shortage prevalence dismissal probability warrior motorist. Discover cancer gate spell. Initially investigator opt cannot colourful production. Shoot capable concentration jail utilize. patent disastrous 2022-08-10T06:28:28 534581.4191 desktops bigint_col 2012-10-22 [9.955149870448799e+17, 5.458963138726759e+17] ["kThacn", "iqAYojfNU", "OmajtKJm", "qfWesYxCEJWqTS", "ZrwOMXcp", "DPeldCDbfKlJuGZKYwTK"] +1 11 809290623 686793101332632 true 4.71326496E8 3.3112444749135328E16 Dream biology decline modification councillor large-scale stair. Perfectly possess abundance. Constitution carbon amazed declare opposed lesser supervisor. Accountant furious consolidate atmosphere pin direction this. Creator illustrate yell viewer ankle. disclosure old appetite \N 231822.0509 desktops bigint_col 2022-02-23 [] \N +1 12 720480298 439676960593506542 true 8.7667955E8 \N Revision flying spelling steer vicious apparent. Tendency follow must adhere reply festival. half title stupid virtual 2022-08-26T05:05:27 644585.1705 desktops bigint_col \N [5.799499253822285e+17, 6.977374510334419e+17, 2.8792573287467094e+17] [] +1 12 872242197 39372924164800285 true 4.02162976E8 5.4119588683596512E17 Bulk fundamental disagree bench past acknowledge. Mercy toxic sing. might chef \N 388501.8604 desktops bigint_col 2022-08-29 [] [null, "hYetKKEDJm"] +1 14 726053880 161015669819134929 false 4.9883168E8 4.3323385209889466E17 Team always illusion advantage could. Instrumental stability seven manager care declare. Jewellery navigation bee creature violence bathroom or. Routine identity application presume him. Pan emerge temporarily. counter consciousness study 2022-08-08T13:52:40 366074.9783 desktops bigint_col 2013-02-09 [1.7713806707666048e+17, 3.7383671495565165e+17, 6.259419911350532e+17, 2.5287201471209952e+17] ["vaSfuLXpBIzKBLnaBd", "DXbG", "OaCJCGdl"] +1 15 98067009 107002720698930270 true 1.01148912E8 5.7735895489617459E17 Single misleading neutral unknown footage above kind. Walk insect harvest hair suspicion traffic. Slap endure text instrument naval again. \N 2022-08-13T01:39:09 332059.6117 desktops bigint_col 2013-04-09 [7.035787745720701e+17] ["wBJJPblSffQaeWC", "OwhqU", "EqotFomTShNObQcm", null, "nMRD", null, "yhtbbFbanWuSmjycpgLj", "ALyVKaolMG", null] +1 20 86450486 128585355387472229 true 3.85727264E8 8.0578324815203955E17 Renew review concentration child pronounce nod emergence. Due travel letter red. private each rocket hers 2022-08-11T04:59:32 644172.2438 desktops bigint_col 2017-12-20 [] [null, "BJWAHVFXmUmTgeJ", "yGZpeeuEAKC"] +1 20 907476179 51144570531813048 \N 1.14442112E8 6.3573879782317082E17 Guilt decide electronic expectation potential outdoors. Magnitude unable furious. healthy bike controversy 2022-09-03T09:08:57 368422.0700 desktops bigint_col 2021-06-18 [9.10700912260912e+17, 4.405327840725512e+17] [] +1 28 837555130 230916677115824233 true 5.4569421E8 2.70471322007323488E17 Cousin artificial abandon custody beam provision. Chemical abroad human. Arms young see. handy initial neat navigation 2022-09-03T17:55:12 524469.0578 desktops bigint_col 2018-09-04 [3.855014125096763e+17, 2.1039308993257866e+17, 9.31498606282679e+17] ["lgtZYBShnyC", "nRQjgF", "aCVIMQpZuYqOig", "lbVNNHeRK", "HsOO", "IJYFhh", "MDthVyeZpCjF", "sMnbIiEFr", "CyLO", "UAWGAZiQDzkpQBYD"] +1 28 838335879 382950133409826292 true 6.2795968E8 3.7588441507785466E17 Evaluate locate guitar frightened. Premier warrior organized buy eat. About die rock creep. civilization discretion desert connect firework 2022-09-02T22:27:57 479988.5725 desktops bigint_col 2020-12-16 [7.804613874622225e+17, 1.8854381373872086e+17] ["Mfwddzogb", "LGdzf", "gEQy"] +1 31 \N 741341588454293823 true 3.86187136E8 4.6610951470341754E17 Aspiration deployment tough make-up. Listing bride hole revision criticize ray sponsorship. Former whereby vanish strain typically. Transition lost erect settle provoke fate. Action atrocity wildlife weird helpful inclined. vary matching \N 389141.8253 desktops bigint_col 2019-04-30 [2.6031378443226595e+17, 9.016824926976108e+17, 5.316609472515438e+17, 3.669648424112748e+17] [null, "EEXcpNyLcwWMyfI"] +1 36 260311723 663342854057084267 false 7.9524992E8 \N Surround characterize wonder. Lady inner cost. Succession clerk increase religious governor squad. Reconstruction commission immediate bit hurt debate. variable contractor female pastor 2022-09-03T19:12:06 947575.7788 desktops bigint_col 2020-12-23 [] ["qiqjac", "SRZCFcqIelw", "rPDODktyX", "QYIZgxOhzLpMIKkzYik", "yfrsvMThiRjyFUypbZ", "WaVFzUSxW", "PRPQuClAUEdqbI", "WFA", "vbQoqNFLfaWRFVcrDX", "lfW"] +1 40 706858385 86990101748729070 true 5.4196077E8 6.9602303815701107E17 Resemble detection wood give launch distinction. Per attention deliberate mum. Modification segment foreigner complement conflict. Travel historic over advertisement. Baby recover soft satisfaction relatively. litter alone 2022-08-08T20:38:09 922369.7339 desktops bigint_col 2014-11-13 [5.85367882988869e+17, 6.078596726647018e+17] ["mDBdSwBdKdvhBaComWk", null, "rpVrUUrGMTtJ", "RYTzZRieSBWkDMiJF", "FapEEnbaHD", null, "Gxqzfggamjd", "DMd", "SFMJCivDFFiXOak", "reEcSR"] +1 56 563643370 \N false 7.2356896E8 8.2856413152776218E17 \N since super fare wipe 2022-08-14T17:47:58 105736.4432 desktops bigint_col 2012-10-25 [9.610180903194888e+17, 7.288131679313782e+17, 8.355045939461916e+17, 3.4987201052222304e+17] ["hoZwRj", "cMhAJCpHrUZQ", "ocYAxs", "gGVqSQewuauBCUkvzr", "yckCKXxVhrdJp", "eZCHD"] +1 58 968282995 675983799610076771 true 1.7376632E8 1.67326199513268896E17 \N \N 2022-08-11T17:23:24 136928.4456 desktops bigint_col 2016-03-24 [6.426187514709601e+17, 8.332563760183391e+17, 5.7197719695391334e+17] ["rpBteuiLfmGcIIqP", "vBeWoaqiXYckauaJ", "kUepRQOLvwqc", "kbIBPMSLEgpaACkdhI", "nBQuR", null, "RPFYYfTTHqI", "iflOfcPV", "qMIr"] +1 63 \N 122703346364718870 false 9.4316915E8 7.1076074324903078E17 Found illegal battery exam originally depressing profound. Too despite pot old go apparatus description. Goal consensus update take. beneficial lunch troop restrict 2022-08-26T10:00:47 339452.6832 desktops bigint_col 2017-10-07 [9.981882027272243e+17, 6.346875650940211e+17] \N +1 68 317879314 979790364239098905 true 2.34733152E8 9.8859784125654656E16 Procedure can2 dialogue. Why promise wing carefully functional. Enthusiast active chop reality eventually thirsty climb. Crystal rule nearby pop disagree picture. special phone immense \N 517206.9006 desktops bigint_col 2019-07-10 [] \N +1 73 \N 41268263353414053 true 4.40259392E8 \N Accumulate expand deadly danger scheme acute privatization. Touch county independence. Demand medication estate high-profile amid explosion. encompass icon end 2022-08-14T04:16:56 544035.6731 desktops bigint_col 2013-05-15 [3.104792154106638e+17, 3.736110301707989e+17, 6.324875022200906e+17] ["HPImt", null, "uoCqzWAZbfyPT", "gjAycoPu", null] +1 77 238055689 703523995378749126 true 5.8739328E8 \N Obesity significant seemingly december. Mutual handling brush. Respective outlet apparent carriage condition. Deliver rarely message magazine reward. quotation energy heighten 2022-08-27T03:36:31 985381.2251 desktops bigint_col \N [] ["rDSkRmOZxFm", "rCiLqodvJUnZGuCl"] +1 83 797661957 837249969127716283 false 2.6391996E7 3.054802436133497E17 Prevalence meaning spell brilliant bonus control dispute. Harm spark lie1 example tradition physics cheer. Funding high store halfway stimulus exhibit. Cute costume mysterious contempt mainstream. Buck kind barrel local garden. ought sporting 2022-08-16T23:29:46 310008.9678 desktops bigint_col 2020-06-30 [6.145991991429664e+17] ["bxhXIyxPDLd", "jKspKbihIfcg", "hyi", "RiYudALJAzFBAM", "sCUkfo", "XDQE", "alzgvO"] +1 85 741388831 956390649934952628 \N 7.6506342E8 7.8871090083722931E17 Bureaucracy accordingly landscape stark predict. Odds perspective cater miner coloured. Torture riot expire schedule. Sacrifice unprecedented denounce round. biology idiot 2022-08-08T04:26:20 \N desktops bigint_col \N [6.901511195860168e+17, 3.859761682785134e+16, 6.169197677431471e+17, 3.125001437485673e+17, 4.467898541676541e+16] [null, "yTfkbpmIecYwPF", null, "NFxcyq", "jhIslvprFgnYc"] +2 \N 512348397 556071212294545151 true 8.6100269E8 9.4859583110848256E17 Pledge trainer hang visit. Bye dad intake. Aspire speculate even highlight mayor portray. Cottage tune march. Spell formulate religious reside. tape lifestyle coverage 2022-08-21T10:30:33 469496.0119 desktops bigint_col 2021-08-08 [8.834857266908703e+17, 9.773573805703795e+16, 1.5345766649441316e+16] ["kVToBxzgeDlDCD", "QqKB", "OXEsmVBfWhKQPywUXj", null, "chffMKypYJZDQD", "kLyKQbchdAccV"] +2 \N 605386450 161034749716124783 true 5.3331568E8 8.483662331540023E17 Framework define container tomorrow disruption uniform soccer. Chocolate entry robust constitutional heating. Silk temporarily claim survivor flavour grant rental. Nobody carefully coffee adverse initial choice cream. Install diversity drown throughout magazine importance. lucky november 2022-08-10T10:45:05 915079.5564 desktops bigint_col \N [3.4137894979448314e+17, 5.6916409629776275e+17, 9.175450825189115e+17] ["YNUoOphUUFih", "anEahlvkTFWoemt"] +2 12 597868450 970898867721721145 \N 7.2001677E8 \N Go bank butter surprised broad mainstream photographer. Extremely noble fabulous tall. Range clean library poet. Cream prince terrorism. Threat applicant well worst hopefully. problem supervise vital theory 2022-08-30T21:48:47 968233.9679 desktops bigint_col 2017-09-10 [3.2150164349149677e+17, 5.6837981064709496e+16, 1.1855546027353826e+17] ["ExUpe"] +2 12 778351598 178745089173303799 \N 1.87834704E8 4.0492247739101331E17 Stark readily word precedent trillion yesterday. Terrain skill harassment supporter bank competence. Critically enthusiasm panic. compensation pleased bury enter figure 2022-08-09T01:15:27 862233.3268 desktops bigint_col 2015-11-12 [4.059834315213429e+16, 9.949490374303972e+17] ["LpCxUwhIAHb", "OlxTdIU", null, "MTKAIFaSjnlwlsZ"] +2 13 284375569 313022208823056690 true 8.4682362E8 1.80727977302818752E17 West appealing heavy workout next. Security gentle chairman control indicator. deploy vice two programme 2022-08-08T13:15:19 499389.8483 desktops bigint_col \N [2.6052619610612326e+17, 1.6511987897783744e+16] [] +2 25 400500070 943655591782324457 true 4.70849024E8 8.589732873557929E17 Mercy welfare snap definition paint laser. System revive stumble highlight balanced gaming bombing. artificial appointment trigger horizon win \N 800687.0791 desktops bigint_col 2013-01-25 [6.036467426448096e+17, 2.1130489046972413e+17] ["hrayImCOtXikHqCGfBih"] +2 28 515357905 487459024969808193 true 2.2465104E8 9.8016470740501632E17 Must ease comprise drain specialize frighten wine. Mum pronounced quickly resist fail. Even stall marker. Not according constitution judgement. Explicit elaborate supervision ski feeding plane. heating prescribe shaped end 2022-09-03T07:01:54 953756.2917 desktops bigint_col 2015-05-15 [] ["JWybUEoqerKOEvTGMhvc", "JysdjBlWQKLcvIglA", "XNeuEIVeYzRpy"] +2 29 726080366 398808167349063190 false 3.36853632E8 7.8843413319848051E17 Wholly phone succession coat criticism. Analyse overlook leg violation home vessel. Further careless coast. Mutual cost exam correspondence mail despite. unconscious rating 2022-08-13T14:26:23 \N desktops bigint_col 2014-05-17 [2.2106660943234602e+17, 3.6934195934907565e+17, 8.861349520455558e+17, 6.523538011424809e+17, 3.788881047669972e+17] [] +2 32 227944144 699950355366195722 true 4.9220144E8 2.56063632494003872E17 Favour ray funeral accountable comment disaster. Warming fancy interactive signature. solo cycle 2022-08-15T08:47:53 288314.6424 desktops bigint_col 2017-12-22 [1.6780404003026906e+17, 9.205489892961844e+17] ["kTxHOdfJrwSi", "WthIZIQSgYFf", "cdMZPCeIjXIkgNOqkA", "pFLNZyW", "uOSVSHhrB", "SYgmtTJZLMzTMMDbauOc"] +2 33 425838871 943495121042472021 true 6.5801453E8 8.0715585698720205E17 Within bury ground span artificial popularity brother. Destroy successive meal attendance ban leg newspaper. Bend clinic especially. \N 2022-08-30T03:31:04 255834.5576 desktops bigint_col 2015-01-31 [9.816669775583758e+17, 8.183866158337096e+17, 3.614964995764407e+17, 6.51351375624206e+17] ["toKBkrCbdOZbesb", "eZT"] +2 33 621411734 223784268032178682 false 9.9321664E8 1.14314709029271536E17 \N feminist neither 2022-09-04T23:04 883603.3353 desktops bigint_col 2021-09-14 [6.219857054964748e+17, 7.51919542330081e+17, 4.981306541043197e+17] ["vSmAJAygtHrxHvYO", null, null] +2 43 8888148 838033351460227375 false 2.53972448E8 \N Rifle headquarters infect. Assistant below texture yet fork various. outlet mad worst enthusiasm 2022-08-19T01:19:19 105639.1166 desktops bigint_col 2013-09-07 [4.986002691572474e+17, 1.2809228753062507e+17, 5.0491976758581024e+17, 1.5552598850632848e+17] ["kWlBbSGSrQBQflrWAm", "mPnbBog", "kPdLH", "ymeCYO", "tqsTaa", "tnrangtfjQRU", "LTSzBEcbxwwNB", null] +2 44 191401327 671963452353742152 false 1.3778736E8 1.83243227484724992E17 Obey dead profitable. Interpretation anniversary joy neighbouring edit electric tidy. Copyright preservation stark deadly amid reminder mobilize. Among choir fork bored. Mortgage disturb long-time professional tongue. mathematics again \N 360637.6705 desktops bigint_col 2019-10-03 [7.77089975109847e+16] [] +2 46 650434655 39848157600020446 true 6.9146586E8 3.7060518421467648E17 Communicate club specialized recession kick legend. Stake insult early fluid ocean integration compound. Associate economics psychiatric beneath relaxed. several trait homeless pose 2022-08-10T23:20:59 403069.5119 desktops bigint_col 2020-03-09 [1.2093856458728192e+17, 1.3086043347585107e+17, 1.803255161213063e+17, 7.679491592398975e+17, 4.931067289222262e+17] ["FaCQGnfOPaUFEUrPVVpw", null] +2 50 760146996 480450791207697069 false 255391.28 3.9309384934358093E17 Broadcaster trailer twice diagram mountain phase. Of conference carpet invisible. endorse function 2022-08-21T01:11:11 636137.6081 desktops bigint_col \N [1.8884955224866605e+17, 5.598730802774882e+17, 4.158846805828165e+16] ["guIgqkPI", "onBpJgKTcUDh", "uESAiBGXJVA", "bLVptq"] +2 51 724892870 772630919179671999 true 7.2187706E8 2.26274676625050752E17 Sponsorship annoyed awful. Forth twist flight. Super stereotype nurse central. Trio dominance suspicious mechanic commentator warning. \N 2022-09-03T06:49:51 316122.7734 desktops bigint_col 2017-02-12 [] ["XRyNrLG", "mEbfBWAWcUcp", "KTvC", null, null, "BLLXRBys", "IFvSWbvmGQuIko", "BnlaBuIRcpXhrPqpV"] +2 62 947714244 451589400328043377 true 3.4991712E8 6.5850654142658534E17 Important dog constantly capable architect. Strongly strictly industrial creature temporary sufficient people. June connection fabulous. Motivation curriculum offence file tear1 true. July perception arena innovative. verse comprehensive war kingdom 2022-08-29T20:42:46 \N desktops bigint_col 2013-03-23 [8.553412772762889e+17, 5.227546601096389e+16, 6.754153329941062e+17] \N +2 66 \N 362061181042516667 \N 9.2758816E8 1.474065095112208E17 Boost tax identify organized processor wear. Than space identify. And scene horse. sale quota gun closed faith 2022-08-13T12:18:06 668437.3033 desktops bigint_col 2014-07-06 [9.125392280363515e+17, 5.451215191556551e+17, 2.877125037775009e+17, 1.2170082291089424e+17, 2.8477499789790096e+17] ["SKQpnBUqkgx", "hLuKeuoj", null, "kWdBDQbLMvhC", "qfZlNdoSPtewRM", "RcUHf", null, "wBRdsnuXf", "YTMUUkeblcUwHAS", "SffrfiBYbMhkICHSEkF"] +2 75 691337321 868250294959268172 true 7.8525376E8 \N Bedroom acquisition irrelevant cope parking charm call. Bright freeze corporation. Opposed net foreigner train. Infect seriously cycle expect. \N 2022-08-18T09:57:10 338665.2099 desktops bigint_col 2021-03-20 [7.231248276676792e+17, 5.276079075450041e+17, 6.5753225426992024e+16] ["ChM", "jzvpHVkhKJVM", null, "HabybUuFiIUxYQOmN", "axhqXUxkxFqiOojcq"] +2 81 \N 388342099454962780 false 9.1951021E8 3.1236464741799296E17 Towel disadvantage reveal major deficit ethic. Notebook please pay. Fixed bound push downtown contract. Memorial just currency humble pad gather rate. Installation dilemma freedom structural choir album. ensue courtesy 2022-09-01T21:50:30 186208.2738 desktops bigint_col 2018-12-01 [9.063784429530822e+17, 6.977872624277775e+17, 7.721732368138504e+17] ["ujeLWtyDmcBNnfiBEhyx", "lbCLmSFUh", "TjrDQcJtXJoBaK"] +2 83 12394764 675592792500868694 true 3.2378528E8 7.7310517741791206E17 Cotton aspire stick. Theirs brilliant crystal. action withdraw 2022-08-19T16:37 604330.4268 desktops bigint_col 2014-06-03 [9.524860908721682e+17, 3.4250654785536506e+17, 5.135658031439463e+17, 4.8666548104110784e+17] ["VeFWyGrnoNn", "NRONgxhotn", null, null, "ZWYKWoXhRyPnNcWJeYJu", null] +2 87 369321831 260373492213221036 true 1930301.2 6.9965213797235238E17 Nursery veteran seat sixteen feminist basically. Obvious successive marginal laptop aggressive remains meaning. T-shirt passion advance native current utterly. Republic soup vanish likewise motivation awkward. Three gift satisfy once short. political stumble theme allocate 2022-08-11T16:35:33 890894.1055 desktops bigint_col 2019-12-04 [9.821996086294243e+17] ["kKJINHjiotL", "ThdbODAnnUIpZli", "PkoolKnpBtZWyjt", "bvfNYKvuoyQmJATvBFNN", "NPxPf", "kPOpaqrey", "CkZ", "rhIfNqyVZlpBBwxMJOQ", "yijAtQDlNX", "bZZyrkbRjFqeoTyq"] +2 87 49257151 652600171802393243 true 7.5004333E8 3.1997831048427386E17 Ship tone section indoors successor fare deadline. Prisoner breed clothes. Collect popular price seriously eligible. Equipment obvious manage discretion addition. Situated acre sample convey priest shop. effectively foreigner material 2022-08-25T19:33:10 658743.1470 desktops bigint_col 2020-04-09 [1.3796744856031629e+17] [null, null, "xCIEDCLIIB", "oMrXxHBPgjxSYsRQMSDe", "nUptFJynf"] +2 87 815222915 439317556995246224 true \N \N Laughter soccer survival attain receipt. Start body hire stake. \N 2022-08-08T09:41:33 375983.1712 desktops bigint_col \N [1.7520331452734906e+17, 8.218320637011135e+17, 5.102686480109678e+17, 5.3854930022541715e+17] ["wJOcTZvWYVmdKAI", "OLfIAFJsmKk", "onrRnpBxuDJOyb", "HvGOLYbSWSioTFpJ", "gQgHurYxvybeC", "bTkonbxQEh"] +2 90 \N 81538723103026803 false 9.8741286E8 7.5386138519914906E17 Massive course height. Breath total physician. Holiday thank organization translation log. Figure transit abstract. Query executive institute pill equipment definite legislative. vibrant physical green complement altogether \N 496008.6909 desktops bigint_col 2021-08-15 [1.1563862484685605e+17, 2.208686748728399e+17, 3.4907002307698164e+16, 2.4108551188340698e+17, 9.985507838875311e+17] \N +2 90 174219368 674822431440767844 false 6.5953984E8 5.021006591370041E17 Backdrop participation great treat subsidy delicious council. Inability garden heat join technological assault petition. racial tide big infect 2022-08-08T23:15:15 341624.6584 desktops bigint_col 2018-10-04 [7.271072731686944e+17, 8.850691215139868e+17, 4.8463435466539315e+17, 7.25837216316081e+17] [] +2 98 \N 435601208177882399 false 1.1480215E7 7.2935720946187264E17 Enrol assessment tactical. Tongue online combine between. Temporary representative lie2 treat. Consecutive genuine fierce. consent distribution 2022-08-20T08:38:16 822616.1952 desktops bigint_col 2019-07-08 [1.5952729308323066e+17, 7.704485945731784e+17, 9.562956777912644e+17] ["LXOPWnyYizgmvj", "DTNIEBgcpjqDMGd", "OffmIClBBYi", "fMFpVfjclRTjx", "IJtTsiNJ", "TNqlGaPqnszbWpFmBxmL"] +2 98 188402749 1757547065004506 true 8.2675984E7 4.7484333110575418E17 Bishop robot capability prosecution level generally feeding. Scratch each drink. Duo major address escape. Sport shooting standing word robbery. Care rescue reduction productive. swear paragraph 2022-08-08T13:45 290677.1033 desktops bigint_col \N [7.46266874313248e+17] ["CnlIyCMVmigoDrhTuOp", "KvmGQHtpEiYDSXqMz", "KgCoET", "zOas", "SZPuCYbHQWeRcxmuynP", "mjdWpzTQcQy", "JWOXDHpZXmvjxL", "vQhkWMkDphjvCy"] +3 \N 783387927 833784751127412839 false 6.1395981E8 7.2964677883635776E17 \N soup voluntary ridiculous atmosphere liberal 2022-08-26T03:45:45 505553.1253 desktops bigint_col \N [8.846038446769828e+17, 8.355716311748035e+17, 2.5798603786894893e+17, 1.6821654273579677e+17, 9.633949134404147e+17] [null, "XGlJHIwttdCzqIVfWCmH", "XRvnqGd", "dLohrer", null] +3 11 452267632 180732776652068136 true 8.234185E8 5.0673167295151014E17 Subject pop reasoning annual release. Voluntary frightened counsellor edge demonstration congregation ashamed. Permanent exist dismiss. Laptop occasional casualty hundred. role tradition fluid hypothesis 2022-08-25T22:58:22 944476.9648 desktops bigint_col 2022-06-11 [] ["psilFGentDltGyjJ", "XhtWxdxoHdbG", "ahNRjrflBsPI", "cAubMaYWW", "KcpzcsaBgnyLOrChpSy", "ckBfKZeanAzCg"] +3 26 327351446 453711707916208854 false 1.3737296E8 5.3344294506759917E17 Legislation immune environment benefit. Blessing talent recovery. Heating cartoon lucky overlook texture noisy squad. Loom finger dealer circulate. snow fault 2022-08-27T19:27:19 577975.6390 desktops bigint_col \N [] ["SLPbAJYqM", "ImDnN"] +3 29 556030687 131420083813462395 true 6.3668006E8 1.68308855103597504E17 \N \N 2022-08-30T09:24:47 883705.3697 desktops bigint_col 2017-11-11 [6.742358467833588e+17, 7.746370686491464e+17] ["fZIwBBispsCxyXmN", null, "lJhklzHS", "evvakkUnLs", "jLiRDsCOdTpJrimosUfM"] +3 31 545321127 849437036516380697 false 6.7927539E8 8.889170476513504E17 Extraordinary graphic modern commissioner you treaty. Extensive humble carve. dot integrity friendly shell 2022-08-08T13:54:08 470633.1523 desktops bigint_col 2018-01-19 [8.45513368465698e+17, 2.138306506876291e+17, 6.729663167160476e+17, 1.1214072770538208e+17] ["HKHOIHUvShdJJSgr", "TncPINpiMUcG", "NvraQjiGStRHkWiZuyy", null, "bpDUJCQuTKlpyvQDJl", "xhkFefWnbE", "yifisHJugRi", "NmM", "ansFopZVwPKESg", "UqMiLmJjuBqEFGJaMsFu"] +3 31 637297379 681740620949095717 false 3.72037984E8 1.5311977505834896E17 Universal bureaucracy hello critically. Notorious questionnaire convincing thoroughly impact machine important. Rest practice tackle collapse. Steam limited cognitive. \N 2022-08-18T00:10:36 978275.4508 desktops bigint_col 2015-11-01 [6.730618515485289e+17, 7.211752076302418e+17] ["pXjW", "GZx", "LpfBOpdkIIDtzvwgbH", "XQnIEJQasHJuMEkDQ"] +3 33 416890308 864968854877102244 true 3.57917696E8 \N Wedding direct nevertheless tobacco advertisement adopt. Link tell plot. loan settlement bind cluster specialized 2022-08-17T18:36:23 757206.2270 desktops bigint_col 2020-11-29 [2.7678152423529523e+17] ["rDipxjS", "eWEYwMfsjQjHUSUr", "SziEffzHJLcLSpsPvYXT", "sZSztAbVxwRkyrEYDEHT", "sjRQSTgFPRtFmuTbwA", "FUffvrlx", "xotJqXwb", "KEEFKcViaiWSbYPMp"] +3 38 568808594 82321681584796394 true 4.78516768E8 2.43156758463730528E17 Centre noise patent tribunal neutral frequent. Occurrence judicial world art bounce. precision resign 2022-08-29T15:24:31 883708.1258 desktops bigint_col 2016-09-19 [3.080566843096494e+17, 7.818941860140127e+17, 3.048632989978904e+17, 1.2902961399744917e+17, 5.5187143156873e+17] [] +3 52 909991391 827842515928592069 true 1.18878936E8 6.7175271888105011E17 Helpful casino stair relevance wealth smash tape. Spring alert reduction merge lead1. Morning second1 description imprison language combination. Report capital unify normally rare. bonus confine prosecutor be dense \N 377193.7087 desktops bigint_col 2013-09-09 [] ["RRMxHGqkj", "fuFWB", "GbcPzzfpCQiBdTJfopKm", null] +3 57 482875653 526266436633468708 false 4.61775136E8 3.2053274668533485E17 Interface convince breakfast sufficiently dilemma. Rank burden boss press. Busy legend sing option. Objective analysis rental skiing transportation mobile get. Print whip unacceptable float spectrum extensive. doctor handy everyone mobilize deny 2022-08-26T02:59:56 115518.8019 desktops bigint_col 2016-09-27 [1.3560043005574606e+17] \N +3 58 352921058 648712597438227479 \N 3.3095744E8 7.7999726469300518E17 Intelligent crude worthy therefore birthday. Productive impression ecological thick. Detailed hostage tourist argue passive congratulate. Onion human terrorism tomorrow body media altogether. Kingdom lottery threat ambulance old-fashioned. it cigarette committee \N 324831.2649 desktops bigint_col 2013-12-27 [7.666249082433514e+17] ["TVbWbFUf", "iZPWCWemhEtgxWfk", "NyYOyTdBxloWNiBTiyP", "nqPSQlMUqireilv", "EicBNzLPCPNPAG", "MMpkckNiTyFug"] +3 65 318259243 788429562626231382 true 2.53879552E8 5.2861201434808768E16 Visitor trend hydrogen. Privilege relationship actor entertaining exposure legendary. Congratulate online theoretical whereby. Nineteen counsellor reasonably stretch legend. position employer 2022-09-03T12:12:11 608134.9893 desktops bigint_col 2015-05-19 [5.593220113166346e+17] ["VarUrIsYc", "pGFGIcKER"] +3 77 \N 899510136394242816 true 1.05334736E8 8.0177537887940186E17 Weight goal presidency loss. Settlement unveil do1 company carbon wealth. preach collective booking 2022-08-28T05:25:05 673575.5591 desktops bigint_col 2015-02-04 [4.0265714573684e+16, 3.6374623436062803e+17, 9.360348023765316e+17] ["CVc", "QVvmaPaOSUySeNcVA", "NYWhCcvGZkzhEJlQKRsU", "QsUiaIpIpeILEKAKlsKy", null, "iDCqoRFpQIxWCPLx", "qMIcGN", "ObOJlI"] +3 78 321907112 1365849332566195 true 1.85100944E8 1.08461021524732256E17 Mistake reign storm slowly wound. Still minute1 soft worried amusing record employee. Sand cheer descend serve initial intervene. Institution bean excuse sustainable cancel tribal. Pen coloured contemporary genre predecessor entrance. commentator expense learning 2022-08-25T11:20:08 750498.2441 desktops bigint_col 2022-06-07 [5.511430917788297e+17, 8.424231668826157e+17, 5.0052505348145914e+17] ["ezgdxpC", "vgO", "OMkMCZeILnonBZ", "QyurcM", "xzTmLloMMHqeLDHcC", "lTBEXOjqFmwpCjDIiDOw", "Igu"] +3 78 393296774 707084360255744262 true 6.189552E8 \N Ghost layout teens lemon racing until. Every cheerful kitchen so pop near. Coup dull besides. Call society bath willingness rat upcoming sick. \N 2022-08-25T02:36:50 991688.3946 desktops bigint_col 2015-02-11 [9.391942870795297e+17, 6.644484220856004e+17, 7.609218522018826e+17, 7.100199403632896e+17, 1.0405543615581325e+17] ["zDCQKzRPThjzdqteoA", "zaVw", "ijdefQyGGW", "Sdq", "AqPQglMVu", null, "XtDbqkgyUyoYLYOgQt"] +3 96 230092499 696089533808779510 false 3.80778464E8 1.05022936741982704E17 Hilarious banana enthusiast show. Gathering edge robust drunk truly mortgage. Equation stun receipt. Client resolution november weird devise store. disappointment case 2022-08-08T19:38:23 129371.4518 desktops bigint_col 2014-12-28 [2.178241064662456e+17, 3.118728389707264e+17] \N +3 97 243052497 157769598533078600 true 9.4151667E8 6.872807770432976E16 Shake chop hospital leave. Nest part-time thankfully interested. Household shortage around kind pirate pride spending. consideration pastor 2022-08-13T06:38:15 110254.9084 desktops bigint_col 2015-01-19 [2.504371687444008e+17, 4.9783978018663245e+17] ["jev", "ddhryCfoZgWyNEGZ"] +3 99 778108157 61873080930551778 true 9.8627565E8 \N Technology gene shame stock grandfather initiate cluster. Sympathy including grind right. Colleague pour interference vanish eligible. Campaign fold poison various between. Formal proper prize ray do1. \N 2022-08-29T19:43:46 908171.0209 desktops bigint_col 2020-02-29 [8.289095771326624e+17, 1.9573475208108093e+17, 6.703456714526179e+17, 7.245677936816932e+17] ["zJldV", "BWhpggYvPlozXm", "bAbglLACbIPnJXpyjDIa"] + +-- !desc_4 -- +bigint_col BIGINT Yes false \N NONE +binary_col TEXT Yes false \N NONE +boolean_col BOOLEAN Yes false \N NONE +char_col TEXT Yes false \N NONE +date_col DATE Yes false \N NONE +decimal_col DECIMAL(12, 4) Yes false \N NONE +double_col DOUBLE Yes false \N NONE +float_col FLOAT Yes false \N NONE +int_col INT Yes true \N +list_double_col ARRAY Yes false \N NONE +list_string_col ARRAY Yes false \N NONE +smallint_col SMALLINT Yes true \N +string_col VARCHAR(65533) Yes false \N NONE +timestamp_col DATETIME(6) Yes false \N NONE +tinyint_col TINYINT Yes true \N +varchar_col TEXT Yes false \N NONE + +-- !select_4 -- +5 10 775197769 782563338151880185 true 3.2961472E7 1.991161785844132E15 Width activation annoying. Speaker senator cultivate convention silence price second1. Ok personal formulate princess. Screening debt salary reluctant have circulate exclusion. Shy immigrant trousers fifteen cat opponent. virtue commander away soak cup 2022-09-05T05:48:47 295491.8278 desktops bigint_col 2020-07-13 [9.146835345997748e+17, 6.733257341227244e+17] ["IusVsK", "ccDDSRdDmaEijQ", "yolElQ", "DbubJwBeNUWSvwTP"] +6 10 645804272 655670909733732612 true 6.3032851E8 9.5434476277711334E17 Merge fulfil authentic. Paint honest keyboard wave live2. Involvement fighting suppose freeze investigate hers glass. Marriage celebrity shut philosophical. studio frequency asylum 2022-08-15T14:15 775744.0766 desktops bigint_col 2015-01-02 [] ["cNHvLdsDvCtx", "lwk", "UXJKknrjYcBvgwRLKUT", "UENYKcHqOaDmjbSQo", "mUjCFTiMpKbc", "FgTQKJloNrGXwCYhBRmQ"] + +-- !desc5 -- +bigint_col BIGINT Yes false \N NONE +binary_col TEXT Yes false \N NONE +boolean_col BOOLEAN Yes false \N NONE +char_col TEXT Yes false \N NONE +date_col DATE Yes false \N NONE +decimal_col DECIMAL(12, 4) Yes false \N NONE +double_col DOUBLE Yes false \N NONE +float_col FLOAT Yes false \N NONE +int_col INT Yes true \N +list_double_col ARRAY Yes false \N NONE +list_string_col ARRAY Yes false \N NONE +p1_col VARCHAR(65533) No false \N NONE +p2_col TEXT No false \N NONE +smallint_col SMALLINT Yes true \N +string_col TEXT Yes false \N NONE +timestamp_col DATETIME(6) Yes false \N NONE +tinyint_col TINYINT Yes true \N +varchar_col TEXT Yes false \N NONE + +-- !select5 -- +1 \N \N 757403305318104467 false 3.26199968E8 1.0049111235672792E17 Consolidate iron breakfast inhibit obesity mount hearing. Limitation bite sibling creation between sound. Plus1 layer injury favourable detain. Learn pronounced entrepreneur personnel wool strive. Pose curiosity spite absolutely combination right. \N 2022-08-11T10:09:31 996888.8617 desktops bigint_col 2015-08-24 [5.084045411017597e+17, 3.942856911182207e+17, 8.38109720690003e+17, 5.0079271855467546e+17] ["NRcqedH", "JIkT", "JXw", "JLvj"] desktops bigint_col +1 \N 107386238 108422142707872166 true 8.3058278E8 3.7039169232624294E17 Cut god fade script breakfast delete. Reasoning enthusiast column genetic tolerate cop school. false merge onto 2022-08-18T16:59:24 549352.8096 desktops bigint_col 2015-10-23 [] ["URknu", "WpuINXTasndUKtb", "tSIMVWOtOQy", "nmFV", "QRnFVjUex"] desktops bigint_col +1 \N 653934097 947581034711672436 false 3.82426912E8 \N Sorry shortage prevalence dismissal probability warrior motorist. Discover cancer gate spell. Initially investigator opt cannot colourful production. Shoot capable concentration jail utilize. patent disastrous 2022-08-10T06:28:28 534581.4191 desktops bigint_col 2012-10-22 [9.955149870448799e+17, 5.458963138726759e+17] ["kThacn", "iqAYojfNU", "OmajtKJm", "qfWesYxCEJWqTS", "ZrwOMXcp", "DPeldCDbfKlJuGZKYwTK"] desktops bigint_col +1 11 809290623 686793101332632 true 4.71326496E8 3.3112444749135328E16 Dream biology decline modification councillor large-scale stair. Perfectly possess abundance. Constitution carbon amazed declare opposed lesser supervisor. Accountant furious consolidate atmosphere pin direction this. Creator illustrate yell viewer ankle. disclosure old appetite \N 231822.0509 desktops bigint_col 2022-02-23 [] \N desktops bigint_col +1 12 720480298 439676960593506542 true 8.7667955E8 \N Revision flying spelling steer vicious apparent. Tendency follow must adhere reply festival. half title stupid virtual 2022-08-26T05:05:27 644585.1705 desktops bigint_col \N [5.799499253822285e+17, 6.977374510334419e+17, 2.8792573287467094e+17] [] desktops bigint_col +1 12 872242197 39372924164800285 true 4.02162976E8 5.4119588683596512E17 Bulk fundamental disagree bench past acknowledge. Mercy toxic sing. might chef \N 388501.8604 desktops bigint_col 2022-08-29 [] [null, "hYetKKEDJm"] desktops bigint_col +1 14 726053880 161015669819134929 false 4.9883168E8 4.3323385209889466E17 Team always illusion advantage could. Instrumental stability seven manager care declare. Jewellery navigation bee creature violence bathroom or. Routine identity application presume him. Pan emerge temporarily. counter consciousness study 2022-08-08T13:52:40 366074.9783 desktops bigint_col 2013-02-09 [1.7713806707666048e+17, 3.7383671495565165e+17, 6.259419911350532e+17, 2.5287201471209952e+17] ["vaSfuLXpBIzKBLnaBd", "DXbG", "OaCJCGdl"] desktops bigint_col +1 15 98067009 107002720698930270 true 1.01148912E8 5.7735895489617459E17 Single misleading neutral unknown footage above kind. Walk insect harvest hair suspicion traffic. Slap endure text instrument naval again. \N 2022-08-13T01:39:09 332059.6117 desktops bigint_col 2013-04-09 [7.035787745720701e+17] ["wBJJPblSffQaeWC", "OwhqU", "EqotFomTShNObQcm", null, "nMRD", null, "yhtbbFbanWuSmjycpgLj", "ALyVKaolMG", null] desktops bigint_col +1 20 86450486 128585355387472229 true 3.85727264E8 8.0578324815203955E17 Renew review concentration child pronounce nod emergence. Due travel letter red. private each rocket hers 2022-08-11T04:59:32 644172.2438 desktops bigint_col 2017-12-20 [] [null, "BJWAHVFXmUmTgeJ", "yGZpeeuEAKC"] desktops bigint_col +1 20 907476179 51144570531813048 \N 1.14442112E8 6.3573879782317082E17 Guilt decide electronic expectation potential outdoors. Magnitude unable furious. healthy bike controversy 2022-09-03T09:08:57 368422.0700 desktops bigint_col 2021-06-18 [9.10700912260912e+17, 4.405327840725512e+17] [] desktops bigint_col +1 28 837555130 230916677115824233 true 5.4569421E8 2.70471322007323488E17 Cousin artificial abandon custody beam provision. Chemical abroad human. Arms young see. handy initial neat navigation 2022-09-03T17:55:12 524469.0578 desktops bigint_col 2018-09-04 [3.855014125096763e+17, 2.1039308993257866e+17, 9.31498606282679e+17] ["lgtZYBShnyC", "nRQjgF", "aCVIMQpZuYqOig", "lbVNNHeRK", "HsOO", "IJYFhh", "MDthVyeZpCjF", "sMnbIiEFr", "CyLO", "UAWGAZiQDzkpQBYD"] desktops bigint_col +1 28 838335879 382950133409826292 true 6.2795968E8 3.7588441507785466E17 Evaluate locate guitar frightened. Premier warrior organized buy eat. About die rock creep. civilization discretion desert connect firework 2022-09-02T22:27:57 479988.5725 desktops bigint_col 2020-12-16 [7.804613874622225e+17, 1.8854381373872086e+17] ["Mfwddzogb", "LGdzf", "gEQy"] desktops bigint_col +1 31 \N 741341588454293823 true 3.86187136E8 4.6610951470341754E17 Aspiration deployment tough make-up. Listing bride hole revision criticize ray sponsorship. Former whereby vanish strain typically. Transition lost erect settle provoke fate. Action atrocity wildlife weird helpful inclined. vary matching \N 389141.8253 desktops bigint_col 2019-04-30 [2.6031378443226595e+17, 9.016824926976108e+17, 5.316609472515438e+17, 3.669648424112748e+17] [null, "EEXcpNyLcwWMyfI"] desktops bigint_col +1 36 260311723 663342854057084267 false 7.9524992E8 \N Surround characterize wonder. Lady inner cost. Succession clerk increase religious governor squad. Reconstruction commission immediate bit hurt debate. variable contractor female pastor 2022-09-03T19:12:06 947575.7788 desktops bigint_col 2020-12-23 [] ["qiqjac", "SRZCFcqIelw", "rPDODktyX", "QYIZgxOhzLpMIKkzYik", "yfrsvMThiRjyFUypbZ", "WaVFzUSxW", "PRPQuClAUEdqbI", "WFA", "vbQoqNFLfaWRFVcrDX", "lfW"] desktops bigint_col +1 40 706858385 86990101748729070 true 5.4196077E8 6.9602303815701107E17 Resemble detection wood give launch distinction. Per attention deliberate mum. Modification segment foreigner complement conflict. Travel historic over advertisement. Baby recover soft satisfaction relatively. litter alone 2022-08-08T20:38:09 922369.7339 desktops bigint_col 2014-11-13 [5.85367882988869e+17, 6.078596726647018e+17] ["mDBdSwBdKdvhBaComWk", null, "rpVrUUrGMTtJ", "RYTzZRieSBWkDMiJF", "FapEEnbaHD", null, "Gxqzfggamjd", "DMd", "SFMJCivDFFiXOak", "reEcSR"] desktops bigint_col +1 56 563643370 \N false 7.2356896E8 8.2856413152776218E17 \N since super fare wipe 2022-08-14T17:47:58 105736.4432 desktops bigint_col 2012-10-25 [9.610180903194888e+17, 7.288131679313782e+17, 8.355045939461916e+17, 3.4987201052222304e+17] ["hoZwRj", "cMhAJCpHrUZQ", "ocYAxs", "gGVqSQewuauBCUkvzr", "yckCKXxVhrdJp", "eZCHD"] desktops bigint_col +1 58 968282995 675983799610076771 true 1.7376632E8 1.67326199513268896E17 \N \N 2022-08-11T17:23:24 136928.4456 desktops bigint_col 2016-03-24 [6.426187514709601e+17, 8.332563760183391e+17, 5.7197719695391334e+17] ["rpBteuiLfmGcIIqP", "vBeWoaqiXYckauaJ", "kUepRQOLvwqc", "kbIBPMSLEgpaACkdhI", "nBQuR", null, "RPFYYfTTHqI", "iflOfcPV", "qMIr"] desktops bigint_col +1 63 \N 122703346364718870 false 9.4316915E8 7.1076074324903078E17 Found illegal battery exam originally depressing profound. Too despite pot old go apparatus description. Goal consensus update take. beneficial lunch troop restrict 2022-08-26T10:00:47 339452.6832 desktops bigint_col 2017-10-07 [9.981882027272243e+17, 6.346875650940211e+17] \N desktops bigint_col +1 68 317879314 979790364239098905 true 2.34733152E8 9.8859784125654656E16 Procedure can2 dialogue. Why promise wing carefully functional. Enthusiast active chop reality eventually thirsty climb. Crystal rule nearby pop disagree picture. special phone immense \N 517206.9006 desktops bigint_col 2019-07-10 [] \N desktops bigint_col +1 73 \N 41268263353414053 true 4.40259392E8 \N Accumulate expand deadly danger scheme acute privatization. Touch county independence. Demand medication estate high-profile amid explosion. encompass icon end 2022-08-14T04:16:56 544035.6731 desktops bigint_col 2013-05-15 [3.104792154106638e+17, 3.736110301707989e+17, 6.324875022200906e+17] ["HPImt", null, "uoCqzWAZbfyPT", "gjAycoPu", null] desktops bigint_col +1 77 238055689 703523995378749126 true 5.8739328E8 \N Obesity significant seemingly december. Mutual handling brush. Respective outlet apparent carriage condition. Deliver rarely message magazine reward. quotation energy heighten 2022-08-27T03:36:31 985381.2251 desktops bigint_col \N [] ["rDSkRmOZxFm", "rCiLqodvJUnZGuCl"] desktops bigint_col +1 83 797661957 837249969127716283 false 2.6391996E7 3.054802436133497E17 Prevalence meaning spell brilliant bonus control dispute. Harm spark lie1 example tradition physics cheer. Funding high store halfway stimulus exhibit. Cute costume mysterious contempt mainstream. Buck kind barrel local garden. ought sporting 2022-08-16T23:29:46 310008.9678 desktops bigint_col 2020-06-30 [6.145991991429664e+17] ["bxhXIyxPDLd", "jKspKbihIfcg", "hyi", "RiYudALJAzFBAM", "sCUkfo", "XDQE", "alzgvO"] desktops bigint_col +1 85 741388831 956390649934952628 \N 7.6506342E8 7.8871090083722931E17 Bureaucracy accordingly landscape stark predict. Odds perspective cater miner coloured. Torture riot expire schedule. Sacrifice unprecedented denounce round. biology idiot 2022-08-08T04:26:20 \N desktops bigint_col \N [6.901511195860168e+17, 3.859761682785134e+16, 6.169197677431471e+17, 3.125001437485673e+17, 4.467898541676541e+16] [null, "yTfkbpmIecYwPF", null, "NFxcyq", "jhIslvprFgnYc"] desktops bigint_col +2 \N 512348397 556071212294545151 true 8.6100269E8 9.4859583110848256E17 Pledge trainer hang visit. Bye dad intake. Aspire speculate even highlight mayor portray. Cottage tune march. Spell formulate religious reside. tape lifestyle coverage 2022-08-21T10:30:33 469496.0119 desktops bigint_col 2021-08-08 [8.834857266908703e+17, 9.773573805703795e+16, 1.5345766649441316e+16] ["kVToBxzgeDlDCD", "QqKB", "OXEsmVBfWhKQPywUXj", null, "chffMKypYJZDQD", "kLyKQbchdAccV"] desktops bigint_col +2 \N 605386450 161034749716124783 true 5.3331568E8 8.483662331540023E17 Framework define container tomorrow disruption uniform soccer. Chocolate entry robust constitutional heating. Silk temporarily claim survivor flavour grant rental. Nobody carefully coffee adverse initial choice cream. Install diversity drown throughout magazine importance. lucky november 2022-08-10T10:45:05 915079.5564 desktops bigint_col \N [3.4137894979448314e+17, 5.6916409629776275e+17, 9.175450825189115e+17] ["YNUoOphUUFih", "anEahlvkTFWoemt"] desktops bigint_col +2 12 597868450 970898867721721145 \N 7.2001677E8 \N Go bank butter surprised broad mainstream photographer. Extremely noble fabulous tall. Range clean library poet. Cream prince terrorism. Threat applicant well worst hopefully. problem supervise vital theory 2022-08-30T21:48:47 968233.9679 desktops bigint_col 2017-09-10 [3.2150164349149677e+17, 5.6837981064709496e+16, 1.1855546027353826e+17] ["ExUpe"] desktops bigint_col +2 12 778351598 178745089173303799 \N 1.87834704E8 4.0492247739101331E17 Stark readily word precedent trillion yesterday. Terrain skill harassment supporter bank competence. Critically enthusiasm panic. compensation pleased bury enter figure 2022-08-09T01:15:27 862233.3268 desktops bigint_col 2015-11-12 [4.059834315213429e+16, 9.949490374303972e+17] ["LpCxUwhIAHb", "OlxTdIU", null, "MTKAIFaSjnlwlsZ"] desktops bigint_col +2 13 284375569 313022208823056690 true 8.4682362E8 1.80727977302818752E17 West appealing heavy workout next. Security gentle chairman control indicator. deploy vice two programme 2022-08-08T13:15:19 499389.8483 desktops bigint_col \N [2.6052619610612326e+17, 1.6511987897783744e+16] [] desktops bigint_col +2 25 400500070 943655591782324457 true 4.70849024E8 8.589732873557929E17 Mercy welfare snap definition paint laser. System revive stumble highlight balanced gaming bombing. artificial appointment trigger horizon win \N 800687.0791 desktops bigint_col 2013-01-25 [6.036467426448096e+17, 2.1130489046972413e+17] ["hrayImCOtXikHqCGfBih"] desktops bigint_col +2 28 515357905 487459024969808193 true 2.2465104E8 9.8016470740501632E17 Must ease comprise drain specialize frighten wine. Mum pronounced quickly resist fail. Even stall marker. Not according constitution judgement. Explicit elaborate supervision ski feeding plane. heating prescribe shaped end 2022-09-03T07:01:54 953756.2917 desktops bigint_col 2015-05-15 [] ["JWybUEoqerKOEvTGMhvc", "JysdjBlWQKLcvIglA", "XNeuEIVeYzRpy"] desktops bigint_col +2 29 726080366 398808167349063190 false 3.36853632E8 7.8843413319848051E17 Wholly phone succession coat criticism. Analyse overlook leg violation home vessel. Further careless coast. Mutual cost exam correspondence mail despite. unconscious rating 2022-08-13T14:26:23 \N desktops bigint_col 2014-05-17 [2.2106660943234602e+17, 3.6934195934907565e+17, 8.861349520455558e+17, 6.523538011424809e+17, 3.788881047669972e+17] [] desktops bigint_col +2 32 227944144 699950355366195722 true 4.9220144E8 2.56063632494003872E17 Favour ray funeral accountable comment disaster. Warming fancy interactive signature. solo cycle 2022-08-15T08:47:53 288314.6424 desktops bigint_col 2017-12-22 [1.6780404003026906e+17, 9.205489892961844e+17] ["kTxHOdfJrwSi", "WthIZIQSgYFf", "cdMZPCeIjXIkgNOqkA", "pFLNZyW", "uOSVSHhrB", "SYgmtTJZLMzTMMDbauOc"] desktops bigint_col +2 33 425838871 943495121042472021 true 6.5801453E8 8.0715585698720205E17 Within bury ground span artificial popularity brother. Destroy successive meal attendance ban leg newspaper. Bend clinic especially. \N 2022-08-30T03:31:04 255834.5576 desktops bigint_col 2015-01-31 [9.816669775583758e+17, 8.183866158337096e+17, 3.614964995764407e+17, 6.51351375624206e+17] ["toKBkrCbdOZbesb", "eZT"] desktops bigint_col +2 33 621411734 223784268032178682 false 9.9321664E8 1.14314709029271536E17 \N feminist neither 2022-09-04T23:04 883603.3353 desktops bigint_col 2021-09-14 [6.219857054964748e+17, 7.51919542330081e+17, 4.981306541043197e+17] ["vSmAJAygtHrxHvYO", null, null] desktops bigint_col +2 43 8888148 838033351460227375 false 2.53972448E8 \N Rifle headquarters infect. Assistant below texture yet fork various. outlet mad worst enthusiasm 2022-08-19T01:19:19 105639.1166 desktops bigint_col 2013-09-07 [4.986002691572474e+17, 1.2809228753062507e+17, 5.0491976758581024e+17, 1.5552598850632848e+17] ["kWlBbSGSrQBQflrWAm", "mPnbBog", "kPdLH", "ymeCYO", "tqsTaa", "tnrangtfjQRU", "LTSzBEcbxwwNB", null] desktops bigint_col +2 44 191401327 671963452353742152 false 1.3778736E8 1.83243227484724992E17 Obey dead profitable. Interpretation anniversary joy neighbouring edit electric tidy. Copyright preservation stark deadly amid reminder mobilize. Among choir fork bored. Mortgage disturb long-time professional tongue. mathematics again \N 360637.6705 desktops bigint_col 2019-10-03 [7.77089975109847e+16] [] desktops bigint_col +2 46 650434655 39848157600020446 true 6.9146586E8 3.7060518421467648E17 Communicate club specialized recession kick legend. Stake insult early fluid ocean integration compound. Associate economics psychiatric beneath relaxed. several trait homeless pose 2022-08-10T23:20:59 403069.5119 desktops bigint_col 2020-03-09 [1.2093856458728192e+17, 1.3086043347585107e+17, 1.803255161213063e+17, 7.679491592398975e+17, 4.931067289222262e+17] ["FaCQGnfOPaUFEUrPVVpw", null] desktops bigint_col +2 50 760146996 480450791207697069 false 255391.28 3.9309384934358093E17 Broadcaster trailer twice diagram mountain phase. Of conference carpet invisible. endorse function 2022-08-21T01:11:11 636137.6081 desktops bigint_col \N [1.8884955224866605e+17, 5.598730802774882e+17, 4.158846805828165e+16] ["guIgqkPI", "onBpJgKTcUDh", "uESAiBGXJVA", "bLVptq"] desktops bigint_col +2 51 724892870 772630919179671999 true 7.2187706E8 2.26274676625050752E17 Sponsorship annoyed awful. Forth twist flight. Super stereotype nurse central. Trio dominance suspicious mechanic commentator warning. \N 2022-09-03T06:49:51 316122.7734 desktops bigint_col 2017-02-12 [] ["XRyNrLG", "mEbfBWAWcUcp", "KTvC", null, null, "BLLXRBys", "IFvSWbvmGQuIko", "BnlaBuIRcpXhrPqpV"] desktops bigint_col +2 62 947714244 451589400328043377 true 3.4991712E8 6.5850654142658534E17 Important dog constantly capable architect. Strongly strictly industrial creature temporary sufficient people. June connection fabulous. Motivation curriculum offence file tear1 true. July perception arena innovative. verse comprehensive war kingdom 2022-08-29T20:42:46 \N desktops bigint_col 2013-03-23 [8.553412772762889e+17, 5.227546601096389e+16, 6.754153329941062e+17] \N desktops bigint_col +2 66 \N 362061181042516667 \N 9.2758816E8 1.474065095112208E17 Boost tax identify organized processor wear. Than space identify. And scene horse. sale quota gun closed faith 2022-08-13T12:18:06 668437.3033 desktops bigint_col 2014-07-06 [9.125392280363515e+17, 5.451215191556551e+17, 2.877125037775009e+17, 1.2170082291089424e+17, 2.8477499789790096e+17] ["SKQpnBUqkgx", "hLuKeuoj", null, "kWdBDQbLMvhC", "qfZlNdoSPtewRM", "RcUHf", null, "wBRdsnuXf", "YTMUUkeblcUwHAS", "SffrfiBYbMhkICHSEkF"] desktops bigint_col +2 75 691337321 868250294959268172 true 7.8525376E8 \N Bedroom acquisition irrelevant cope parking charm call. Bright freeze corporation. Opposed net foreigner train. Infect seriously cycle expect. \N 2022-08-18T09:57:10 338665.2099 desktops bigint_col 2021-03-20 [7.231248276676792e+17, 5.276079075450041e+17, 6.5753225426992024e+16] ["ChM", "jzvpHVkhKJVM", null, "HabybUuFiIUxYQOmN", "axhqXUxkxFqiOojcq"] desktops bigint_col +2 81 \N 388342099454962780 false 9.1951021E8 3.1236464741799296E17 Towel disadvantage reveal major deficit ethic. Notebook please pay. Fixed bound push downtown contract. Memorial just currency humble pad gather rate. Installation dilemma freedom structural choir album. ensue courtesy 2022-09-01T21:50:30 186208.2738 desktops bigint_col 2018-12-01 [9.063784429530822e+17, 6.977872624277775e+17, 7.721732368138504e+17] ["ujeLWtyDmcBNnfiBEhyx", "lbCLmSFUh", "TjrDQcJtXJoBaK"] desktops bigint_col +2 83 12394764 675592792500868694 true 3.2378528E8 7.7310517741791206E17 Cotton aspire stick. Theirs brilliant crystal. action withdraw 2022-08-19T16:37 604330.4268 desktops bigint_col 2014-06-03 [9.524860908721682e+17, 3.4250654785536506e+17, 5.135658031439463e+17, 4.8666548104110784e+17] ["VeFWyGrnoNn", "NRONgxhotn", null, null, "ZWYKWoXhRyPnNcWJeYJu", null] desktops bigint_col +2 87 369321831 260373492213221036 true 1930301.2 6.9965213797235238E17 Nursery veteran seat sixteen feminist basically. Obvious successive marginal laptop aggressive remains meaning. T-shirt passion advance native current utterly. Republic soup vanish likewise motivation awkward. Three gift satisfy once short. political stumble theme allocate 2022-08-11T16:35:33 890894.1055 desktops bigint_col 2019-12-04 [9.821996086294243e+17] ["kKJINHjiotL", "ThdbODAnnUIpZli", "PkoolKnpBtZWyjt", "bvfNYKvuoyQmJATvBFNN", "NPxPf", "kPOpaqrey", "CkZ", "rhIfNqyVZlpBBwxMJOQ", "yijAtQDlNX", "bZZyrkbRjFqeoTyq"] desktops bigint_col +2 87 49257151 652600171802393243 true 7.5004333E8 3.1997831048427386E17 Ship tone section indoors successor fare deadline. Prisoner breed clothes. Collect popular price seriously eligible. Equipment obvious manage discretion addition. Situated acre sample convey priest shop. effectively foreigner material 2022-08-25T19:33:10 658743.1470 desktops bigint_col 2020-04-09 [1.3796744856031629e+17] [null, null, "xCIEDCLIIB", "oMrXxHBPgjxSYsRQMSDe", "nUptFJynf"] desktops bigint_col +2 87 815222915 439317556995246224 true \N \N Laughter soccer survival attain receipt. Start body hire stake. \N 2022-08-08T09:41:33 375983.1712 desktops bigint_col \N [1.7520331452734906e+17, 8.218320637011135e+17, 5.102686480109678e+17, 5.3854930022541715e+17] ["wJOcTZvWYVmdKAI", "OLfIAFJsmKk", "onrRnpBxuDJOyb", "HvGOLYbSWSioTFpJ", "gQgHurYxvybeC", "bTkonbxQEh"] desktops bigint_col +2 90 \N 81538723103026803 false 9.8741286E8 7.5386138519914906E17 Massive course height. Breath total physician. Holiday thank organization translation log. Figure transit abstract. Query executive institute pill equipment definite legislative. vibrant physical green complement altogether \N 496008.6909 desktops bigint_col 2021-08-15 [1.1563862484685605e+17, 2.208686748728399e+17, 3.4907002307698164e+16, 2.4108551188340698e+17, 9.985507838875311e+17] \N desktops bigint_col +2 90 174219368 674822431440767844 false 6.5953984E8 5.021006591370041E17 Backdrop participation great treat subsidy delicious council. Inability garden heat join technological assault petition. racial tide big infect 2022-08-08T23:15:15 341624.6584 desktops bigint_col 2018-10-04 [7.271072731686944e+17, 8.850691215139868e+17, 4.8463435466539315e+17, 7.25837216316081e+17] [] desktops bigint_col +2 98 \N 435601208177882399 false 1.1480215E7 7.2935720946187264E17 Enrol assessment tactical. Tongue online combine between. Temporary representative lie2 treat. Consecutive genuine fierce. consent distribution 2022-08-20T08:38:16 822616.1952 desktops bigint_col 2019-07-08 [1.5952729308323066e+17, 7.704485945731784e+17, 9.562956777912644e+17] ["LXOPWnyYizgmvj", "DTNIEBgcpjqDMGd", "OffmIClBBYi", "fMFpVfjclRTjx", "IJtTsiNJ", "TNqlGaPqnszbWpFmBxmL"] desktops bigint_col +2 98 188402749 1757547065004506 true 8.2675984E7 4.7484333110575418E17 Bishop robot capability prosecution level generally feeding. Scratch each drink. Duo major address escape. Sport shooting standing word robbery. Care rescue reduction productive. swear paragraph 2022-08-08T13:45 290677.1033 desktops bigint_col \N [7.46266874313248e+17] ["CnlIyCMVmigoDrhTuOp", "KvmGQHtpEiYDSXqMz", "KgCoET", "zOas", "SZPuCYbHQWeRcxmuynP", "mjdWpzTQcQy", "JWOXDHpZXmvjxL", "vQhkWMkDphjvCy"] desktops bigint_col +3 \N 783387927 833784751127412839 false 6.1395981E8 7.2964677883635776E17 \N soup voluntary ridiculous atmosphere liberal 2022-08-26T03:45:45 505553.1253 desktops bigint_col \N [8.846038446769828e+17, 8.355716311748035e+17, 2.5798603786894893e+17, 1.6821654273579677e+17, 9.633949134404147e+17] [null, "XGlJHIwttdCzqIVfWCmH", "XRvnqGd", "dLohrer", null] desktops bigint_col +3 11 452267632 180732776652068136 true 8.234185E8 5.0673167295151014E17 Subject pop reasoning annual release. Voluntary frightened counsellor edge demonstration congregation ashamed. Permanent exist dismiss. Laptop occasional casualty hundred. role tradition fluid hypothesis 2022-08-25T22:58:22 944476.9648 desktops bigint_col 2022-06-11 [] ["psilFGentDltGyjJ", "XhtWxdxoHdbG", "ahNRjrflBsPI", "cAubMaYWW", "KcpzcsaBgnyLOrChpSy", "ckBfKZeanAzCg"] desktops bigint_col +3 26 327351446 453711707916208854 false 1.3737296E8 5.3344294506759917E17 Legislation immune environment benefit. Blessing talent recovery. Heating cartoon lucky overlook texture noisy squad. Loom finger dealer circulate. snow fault 2022-08-27T19:27:19 577975.6390 desktops bigint_col \N [] ["SLPbAJYqM", "ImDnN"] desktops bigint_col +3 29 556030687 131420083813462395 true 6.3668006E8 1.68308855103597504E17 \N \N 2022-08-30T09:24:47 883705.3697 desktops bigint_col 2017-11-11 [6.742358467833588e+17, 7.746370686491464e+17] ["fZIwBBispsCxyXmN", null, "lJhklzHS", "evvakkUnLs", "jLiRDsCOdTpJrimosUfM"] desktops bigint_col +3 31 545321127 849437036516380697 false 6.7927539E8 8.889170476513504E17 Extraordinary graphic modern commissioner you treaty. Extensive humble carve. dot integrity friendly shell 2022-08-08T13:54:08 470633.1523 desktops bigint_col 2018-01-19 [8.45513368465698e+17, 2.138306506876291e+17, 6.729663167160476e+17, 1.1214072770538208e+17] ["HKHOIHUvShdJJSgr", "TncPINpiMUcG", "NvraQjiGStRHkWiZuyy", null, "bpDUJCQuTKlpyvQDJl", "xhkFefWnbE", "yifisHJugRi", "NmM", "ansFopZVwPKESg", "UqMiLmJjuBqEFGJaMsFu"] desktops bigint_col +3 31 637297379 681740620949095717 false 3.72037984E8 1.5311977505834896E17 Universal bureaucracy hello critically. Notorious questionnaire convincing thoroughly impact machine important. Rest practice tackle collapse. Steam limited cognitive. \N 2022-08-18T00:10:36 978275.4508 desktops bigint_col 2015-11-01 [6.730618515485289e+17, 7.211752076302418e+17] ["pXjW", "GZx", "LpfBOpdkIIDtzvwgbH", "XQnIEJQasHJuMEkDQ"] desktops bigint_col +3 33 416890308 864968854877102244 true 3.57917696E8 \N Wedding direct nevertheless tobacco advertisement adopt. Link tell plot. loan settlement bind cluster specialized 2022-08-17T18:36:23 757206.2270 desktops bigint_col 2020-11-29 [2.7678152423529523e+17] ["rDipxjS", "eWEYwMfsjQjHUSUr", "SziEffzHJLcLSpsPvYXT", "sZSztAbVxwRkyrEYDEHT", "sjRQSTgFPRtFmuTbwA", "FUffvrlx", "xotJqXwb", "KEEFKcViaiWSbYPMp"] desktops bigint_col +3 38 568808594 82321681584796394 true 4.78516768E8 2.43156758463730528E17 Centre noise patent tribunal neutral frequent. Occurrence judicial world art bounce. precision resign 2022-08-29T15:24:31 883708.1258 desktops bigint_col 2016-09-19 [3.080566843096494e+17, 7.818941860140127e+17, 3.048632989978904e+17, 1.2902961399744917e+17, 5.5187143156873e+17] [] desktops bigint_col +3 52 909991391 827842515928592069 true 1.18878936E8 6.7175271888105011E17 Helpful casino stair relevance wealth smash tape. Spring alert reduction merge lead1. Morning second1 description imprison language combination. Report capital unify normally rare. bonus confine prosecutor be dense \N 377193.7087 desktops bigint_col 2013-09-09 [] ["RRMxHGqkj", "fuFWB", "GbcPzzfpCQiBdTJfopKm", null] desktops bigint_col +3 57 482875653 526266436633468708 false 4.61775136E8 3.2053274668533485E17 Interface convince breakfast sufficiently dilemma. Rank burden boss press. Busy legend sing option. Objective analysis rental skiing transportation mobile get. Print whip unacceptable float spectrum extensive. doctor handy everyone mobilize deny 2022-08-26T02:59:56 115518.8019 desktops bigint_col 2016-09-27 [1.3560043005574606e+17] \N desktops bigint_col +3 58 352921058 648712597438227479 \N 3.3095744E8 7.7999726469300518E17 Intelligent crude worthy therefore birthday. Productive impression ecological thick. Detailed hostage tourist argue passive congratulate. Onion human terrorism tomorrow body media altogether. Kingdom lottery threat ambulance old-fashioned. it cigarette committee \N 324831.2649 desktops bigint_col 2013-12-27 [7.666249082433514e+17] ["TVbWbFUf", "iZPWCWemhEtgxWfk", "NyYOyTdBxloWNiBTiyP", "nqPSQlMUqireilv", "EicBNzLPCPNPAG", "MMpkckNiTyFug"] desktops bigint_col +3 65 318259243 788429562626231382 true 2.53879552E8 5.2861201434808768E16 Visitor trend hydrogen. Privilege relationship actor entertaining exposure legendary. Congratulate online theoretical whereby. Nineteen counsellor reasonably stretch legend. position employer 2022-09-03T12:12:11 608134.9893 desktops bigint_col 2015-05-19 [5.593220113166346e+17] ["VarUrIsYc", "pGFGIcKER"] desktops bigint_col +3 77 \N 899510136394242816 true 1.05334736E8 8.0177537887940186E17 Weight goal presidency loss. Settlement unveil do1 company carbon wealth. preach collective booking 2022-08-28T05:25:05 673575.5591 desktops bigint_col 2015-02-04 [4.0265714573684e+16, 3.6374623436062803e+17, 9.360348023765316e+17] ["CVc", "QVvmaPaOSUySeNcVA", "NYWhCcvGZkzhEJlQKRsU", "QsUiaIpIpeILEKAKlsKy", null, "iDCqoRFpQIxWCPLx", "qMIcGN", "ObOJlI"] desktops bigint_col +3 78 321907112 1365849332566195 true 1.85100944E8 1.08461021524732256E17 Mistake reign storm slowly wound. Still minute1 soft worried amusing record employee. Sand cheer descend serve initial intervene. Institution bean excuse sustainable cancel tribal. Pen coloured contemporary genre predecessor entrance. commentator expense learning 2022-08-25T11:20:08 750498.2441 desktops bigint_col 2022-06-07 [5.511430917788297e+17, 8.424231668826157e+17, 5.0052505348145914e+17] ["ezgdxpC", "vgO", "OMkMCZeILnonBZ", "QyurcM", "xzTmLloMMHqeLDHcC", "lTBEXOjqFmwpCjDIiDOw", "Igu"] desktops bigint_col +3 78 393296774 707084360255744262 true 6.189552E8 \N Ghost layout teens lemon racing until. Every cheerful kitchen so pop near. Coup dull besides. Call society bath willingness rat upcoming sick. \N 2022-08-25T02:36:50 991688.3946 desktops bigint_col 2015-02-11 [9.391942870795297e+17, 6.644484220856004e+17, 7.609218522018826e+17, 7.100199403632896e+17, 1.0405543615581325e+17] ["zDCQKzRPThjzdqteoA", "zaVw", "ijdefQyGGW", "Sdq", "AqPQglMVu", null, "XtDbqkgyUyoYLYOgQt"] desktops bigint_col +3 96 230092499 696089533808779510 false 3.80778464E8 1.05022936741982704E17 Hilarious banana enthusiast show. Gathering edge robust drunk truly mortgage. Equation stun receipt. Client resolution november weird devise store. disappointment case 2022-08-08T19:38:23 129371.4518 desktops bigint_col 2014-12-28 [2.178241064662456e+17, 3.118728389707264e+17] \N desktops bigint_col +3 97 243052497 157769598533078600 true 9.4151667E8 6.872807770432976E16 Shake chop hospital leave. Nest part-time thankfully interested. Household shortage around kind pirate pride spending. consideration pastor 2022-08-13T06:38:15 110254.9084 desktops bigint_col 2015-01-19 [2.504371687444008e+17, 4.9783978018663245e+17] ["jev", "ddhryCfoZgWyNEGZ"] desktops bigint_col +3 99 778108157 61873080930551778 true 9.8627565E8 \N Technology gene shame stock grandfather initiate cluster. Sympathy including grind right. Colleague pour interference vanish eligible. Campaign fold poison various between. Formal proper prize ray do1. \N 2022-08-29T19:43:46 908171.0209 desktops bigint_col 2020-02-29 [8.289095771326624e+17, 1.9573475208108093e+17, 6.703456714526179e+17, 7.245677936816932e+17] ["zJldV", "BWhpggYvPlozXm", "bAbglLACbIPnJXpyjDIa"] desktops bigint_col + diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy index 9c460eaf64ae409..e7de4478a425a58 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy @@ -109,7 +109,7 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive") 'file_format'='${file_format}' ) """ - exception "java.sql.SQLException: errCode = 2, detailMessage = errCode = 2, detailMessage = failed to create database from hms client. reason: java.lang.UnsupportedOperationException: Table with default values is not supported if the hive version is less than 3.0. Can set 'hive.version' to 3.0 in properties." + exception "java.sql.SQLException: errCode = 2, detailMessage = errCode = 2, detailMessage = failed to create table from hms client. reason: java.lang.UnsupportedOperationException: Table with default values is not supported if the hive version is less than 3.0. Can set 'hive.version' to 3.0 in properties." } sql """DROP DATABASE `test_hive_default_val`""" @@ -648,7 +648,7 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive") 'file_format'='${file_format}' ) """ - exception "errCode = 2, detailMessage = errCode = 2, detailMessage = failed to create database from hms client. reason: org.apache.doris.datasource.hive.HMSClientException: Unsupported primitive type conversion of LARGEINT" + exception "errCode = 2, detailMessage = errCode = 2, detailMessage = failed to create table from hms client. reason: org.apache.doris.datasource.hive.HMSClientException: Unsupported primitive type conversion of LARGEINT" } test { diff --git a/regression-test/suites/external_table_p0/hive/write/test_hive_ctas_to_doris.groovy b/regression-test/suites/external_table_p0/hive/write/test_hive_ctas_to_doris.groovy new file mode 100644 index 000000000000000..ebf406255a2f6e3 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/write/test_hive_ctas_to_doris.groovy @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_ctas_to_doris", "p0,external,hive,external_docker,external_docker_hive") { + + for (String hivePrefix : ["hive2"]) { + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + + setHivePrefix(hivePrefix) + try { + String str1 = "0123456789" * 6554 // string.length() = 65540 + String str2 = str1.substring(0, 65533) + + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalog = "test_hive_ctas_to_doris" + String hive_tb = "tb_test_hive_ctas_to_doris" + String db_name = "db_test_hive_ctas_to_doris" + + // create hive table + sql """drop catalog if exists ${catalog}""" + sql """create catalog if not exists ${catalog} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + sql """ create database if not exists ${catalog}.${db_name} """ + sql """ drop table if exists ${catalog}.${db_name}.${hive_tb}""" + sql """ create table ${catalog}.${db_name}.${hive_tb} + (id int, str1 string, str2 string, str3 string) """ + sql """ insert into ${catalog}.${db_name}.${hive_tb} values (1, '${str1}', '${str2}', 'str3') """ + + qt_q01 """ select length(str1),length(str2) ,length(str3) from ${catalog}.${db_name}.${hive_tb} """ + qt_q02 """ desc ${catalog}.${db_name}.${hive_tb} """ + + sql """ create database if not exists internal.${db_name} """ + + // ctas for partition + sql """ create table internal.${db_name}.${hive_tb}_1 (id,str1,str2,str3) auto partition by list (str3)() properties("replication_num" = "1") as select id, str1, str2, str3 from ${catalog}.${db_name}.${hive_tb} """ + qt_q03 """ select length(str1),length(str2) ,length(str3) from internal.${db_name}.${hive_tb}_1 """ + qt_q04 """ desc internal.${db_name}.${hive_tb}_1 """ + sql """ drop table internal.${db_name}.${hive_tb}_1 """ + + // ctas for distribution + sql """ create table internal.${db_name}.${hive_tb}_2 (id,str1,str2,str3) DISTRIBUTED BY HASH(`str2`) BUCKETS 1 properties("replication_num" = "1") as select id, str1, str2, str3 from ${catalog}.${db_name}.${hive_tb} """ + qt_q05 """ select length(str1),length(str2) ,length(str3) from internal.${db_name}.${hive_tb}_2 """ + qt_q06 """ desc internal.${db_name}.${hive_tb}_2 """ + sql """ drop table internal.${db_name}.${hive_tb}_2 """ + + // ctas for distribution + sql """ create table internal.${db_name}.${hive_tb}_3 (id,str1,str2,str3) DISTRIBUTED BY HASH(`str3`) BUCKETS 1 properties("replication_num" = "1") as select id, str1, str2, str3 from ${catalog}.${db_name}.${hive_tb} """ + qt_q07 """ select length(str1),length(str2) ,length(str3) from internal.${db_name}.${hive_tb}_3 """ + qt_q08 """ desc internal.${db_name}.${hive_tb}_3 """ + sql """ drop table internal.${db_name}.${hive_tb}_3 """ + + try { + sql """ create table internal.${db_name}.${hive_tb}_4 (id,str2,str3,str1) auto partition by list (str1)() properties("replication_num" = "1") as select id, str1, str2, str3 from ${catalog}.${db_name}.${hive_tb} """ + assertTrue(false) + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Partition name's length is over limit of 50")) + } + + try { + sql """ create table internal.${db_name}.${hive_tb}_5 (id,str1,str3,str2) auto partition by list (str2)() properties("replication_num" = "1") as select id, str1, str2, str3 from ${catalog}.${db_name}.${hive_tb} """ + assertTrue(false) + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Partition name's length is over limit of 50")) + } + + try { + sql """ create table internal.${db_name}.${hive_tb}_6 (id,str1,str2,str3) DISTRIBUTED BY HASH(`str1`) BUCKETS 1 properties("replication_num" = "1") as select id, str1, str2, str3 from ${catalog}.${db_name}.${hive_tb} """ + assertTrue(false) + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Insert has filtered data in strict mode")) + } + + } finally { + } + } +} + diff --git a/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy b/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy new file mode 100644 index 000000000000000..c02f7258d37b9dd --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_ctas_with_hdfs.groovy @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ctas_with_hdfs","external,hive,tvf,external_docker") { + String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + // It's okay to use random `hdfsUser`, but can not be empty. + def hdfsUserName = "doris" + def format = "orc" + def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}" + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String tableName = "ctas_tvf_1"; + def uri = "${defaultFS}" + "/user/doris/preinstalled_data/orc/orc_all_types/p1_col=desktops/p2_col=bigint_col/*" + + // test varchar type as partition column + sql """drop table if exists ${tableName}; """ + sql """ create table ${tableName} + PARTITION BY LIST(p1_col,p2_col) ( + PARTITION p1 VALUES IN (("desktops", "bigint_col")), + PARTITION p2 VALUES IN (("phones", "float_col")) + ) + PROPERTIES("replication_num" = "1") + as + select * from + HDFS( + "uri" = "${uri}", + "hadoop.username" = "${hdfsUserName}", + "path_partition_keys"="p1_col,p2_col", + "format" = "${format}") where tinyint_col < 4; + """ + order_qt_desc """desc ${tableName};""" + order_qt_select """ select * from ${tableName} order by tinyint_col""" + + // test char type as partition column + tableName = "ctas_tvf_2"; + sql """drop table if exists ${tableName}; """ + sql """ + create table ${tableName} + PARTITION BY LIST(char_col) ( + PARTITION p1 VALUES IN (("desktops")) + ) + PROPERTIES("replication_num" = "1") + as + select * from + HDFS( + "uri" = "${uri}", + "hadoop.username" = "${hdfsUserName}", + "format" = "${format}") where tinyint_col < 4; + """ + order_qt_desc_2 """desc ${tableName};""" + order_qt_select_2 """ select * from ${tableName} order by tinyint_col""" + + // test tinyint_col type as partition column + tableName = "ctas_tvf_3"; + sql """drop table if exists ${tableName}; """ + sql """ + create table ${tableName} + PARTITION BY LIST(tinyint_col) ( + PARTITION p1 VALUES IN ((1)), + PARTITION p2 VALUES IN ((2)), + PARTITION p3 VALUES IN ((3)) + ) + PROPERTIES("replication_num" = "1") + as + select * from + HDFS( + "uri" = "${uri}", + "hadoop.username" = "${hdfsUserName}", + "format" = "${format}") where tinyint_col < 4; + """ + order_qt_desc_3 """desc ${tableName};""" + order_qt_select_3 """ select * from ${tableName} order by tinyint_col""" + + + // test String as partition column + tableName = "ctas_tvf_4"; + sql """drop table if exists ${tableName}; """ + sql """ + create table ${tableName} + PARTITION BY LIST(string_col) ( + PARTITION p1 VALUES IN (("Width activation annoying. Speaker senator cultivate convention silence price second1. Ok personal formulate princess. Screening debt salary reluctant have circulate exclusion. Shy immigrant trousers fifteen cat opponent.")), + PARTITION p2 VALUES IN (("Merge fulfil authentic. Paint honest keyboard wave live2. Involvement fighting suppose freeze investigate hers glass. Marriage celebrity shut philosophical.")) + ) + PROPERTIES("replication_num" = "1") + as + select * from + HDFS( + "uri" = "${uri}", + "hadoop.username" = "${hdfsUserName}", + "format" = "${format}") where smallint_col <11; + """ + order_qt_desc_4 """desc ${tableName};""" + order_qt_select_4 """ select * from ${tableName} order by tinyint_col""" + + // test varchar type as distributed column + tableName = "ctas_tvf_5"; + sql """drop table if exists ${tableName}; """ + sql """ create table ${tableName} + DISTRIBUTED BY HASH(p1_col) BUCKETS 10 + PROPERTIES("replication_num" = "1") + as + select * from + HDFS( + "uri" = "${uri}", + "hadoop.username" = "${hdfsUserName}", + "path_partition_keys"="p1_col,p2_col", + "format" = "${format}") where tinyint_col < 4; + """ + order_qt_desc5 """desc ${tableName};""" + order_qt_select5 """ select * from ${tableName} order by tinyint_col""" + } +}