From 420c84935de69d10c21c47b1b1bfd392e2f30b79 Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:08:16 +0800 Subject: [PATCH 1/7] [Fix](Nereids) fix fe fold constant evaluate like function (#37616) Problem: When evaluating like function using fe, it can not evaluating nullliteral correctly Example: Null like "%string%" can not folded to null on fe Reason: Fe fold constant does not deal with like function Solved: Add fe fold constant of like function --- .../rules/expression/rules/FoldConstantRuleOnFE.java | 6 ------ .../fold_constant/fold_constant_by_fe.groovy | 11 ++++++++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 1106df5f7c37f5..d7627d698d6134 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -43,7 +43,6 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.LessThanEqual; -import org.apache.doris.nereids.trees.expressions.Like; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; @@ -403,11 +402,6 @@ public Expression visitOr(Or or, ExpressionRewriteContext context) { } } - @Override - public Expression visitLike(Like like, ExpressionRewriteContext context) { - return like; - } - @Override public Expression visitCast(Cast cast, ExpressionRewriteContext context) { cast = rewriteChildren(cast, context); diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy index a706ee9abf9114..cc63662657c37c 100644 --- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_fe.groovy @@ -19,6 +19,7 @@ suite("test_fold_constant_by_fe") { sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' sql 'set enable_fold_nondeterministic_fn=true' + sql 'set enable_fold_constant_by_be=false' def results = sql 'select uuid(), uuid()' assertFalse(Objects.equals(results[0][0], results[0][1])) @@ -154,4 +155,12 @@ suite("test_fold_constant_by_fe") { res = res.split('VUNION')[1] assertFalse(res.contains("unix")) } -} \ No newline at end of file + + // test null like string cause of fe need to fold constant like that to enable not null derive + res = sql """explain select null like '%123%'""" + assertFalse(res.contains("like")) + // now fe fold constant still can not deal with this case + res = sql """explain select "12" like '%123%'""" + assertTrue(res.contains("like")) + +} From 42a311d3df41832b6327839828b52c096e3e33bb Mon Sep 17 00:00:00 2001 From: zzzxl <33418555+zzzxl1993@users.noreply.github.com> Date: Mon, 15 Jul 2024 18:50:02 +0800 Subject: [PATCH 2/7] [opt](inverted index) Optimization of the initialization process in topn (#37429) ## Proposed changes 1. reduce the performance cost of initialization --- .../olap/rowset/segment_v2/segment_iterator.cpp | 15 +++++++++------ be/src/olap/rowset/segment_v2/segment_iterator.h | 3 ++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index ea5b5ae01b6bb9..6a66ae6fc146b3 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1925,7 +1925,8 @@ Status SegmentIterator::_read_columns(const std::vector& column_ids, } Status SegmentIterator::_init_current_block( - vectorized::Block* block, std::vector& current_columns) { + vectorized::Block* block, std::vector& current_columns, + uint32_t nrows_read_limit) { block->clear_column_data(_schema->num_column_ids()); for (size_t i = 0; i < _schema->num_column_ids(); i++) { @@ -1945,7 +1946,7 @@ Status SegmentIterator::_init_current_block( column_desc->path() == nullptr ? "" : column_desc->path()->get_path()); // TODO reuse current_columns[cid] = file_column_type->create_column(); - current_columns[cid]->reserve(_opts.block_row_max); + current_columns[cid]->reserve(nrows_read_limit); } else { // the column in block must clear() here to insert new data if (_is_pred_column[cid] || @@ -1964,7 +1965,7 @@ Status SegmentIterator::_init_current_block( } else if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_DATETIME) { current_columns[cid]->set_datetime_type(); } - current_columns[cid]->reserve(_opts.block_row_max); + current_columns[cid]->reserve(nrows_read_limit); } } } @@ -2378,14 +2379,16 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { } } } - RETURN_IF_ERROR(_init_current_block(block, _current_return_columns)); - _converted_column_ids.assign(_schema->columns().size(), 0); - _current_batch_rows_read = 0; uint32_t nrows_read_limit = _opts.block_row_max; if (_can_opt_topn_reads()) { nrows_read_limit = std::min(static_cast(_opts.topn_limit), nrows_read_limit); } + + RETURN_IF_ERROR(_init_current_block(block, _current_return_columns, nrows_read_limit)); + _converted_column_ids.assign(_schema->columns().size(), 0); + + _current_batch_rows_read = 0; RETURN_IF_ERROR(_read_columns_by_index( nrows_read_limit, _current_batch_rows_read, _lazy_materialization_read || _opts.record_rowids || _is_need_expr_eval)); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index ae865ddc456950..cb904f21c6ac20 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -221,7 +221,8 @@ class SegmentIterator : public RowwiseIterator { bool set_block_rowid); void _replace_version_col(size_t num_rows); Status _init_current_block(vectorized::Block* block, - std::vector& non_pred_vector); + std::vector& non_pred_vector, + uint32_t nrows_read_limit); uint16_t _evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size); uint16_t _evaluate_short_circuit_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size); void _output_non_pred_columns(vectorized::Block* block); From 55486bbaf7dd4a53ebdb7dc181e0af9a398301c6 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Mon, 15 Jul 2024 19:23:05 +0800 Subject: [PATCH 3/7] [improvement](mow) Add more log on getDeleteBitmapUpdateLock (#37728) getDeleteBitmapUpdateLock may cost too much time, here is an example: 2024-07-08 16:58:43,050 INFO (thrift-server-pool-1|183) [CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock():521] get delete bitmap lock successfully. txns: 27787309217006594. time cost: 10500 ms. So need to add more log to find out why it cost so much time. --- .../cloud/transaction/CloudGlobalTransactionMgr.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index f3e106743d4ae9..727192b4e572aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -714,6 +714,7 @@ private void getDeleteBitmapUpdateLock(Map> tableToParttions, lo } StopWatch stopWatch = new StopWatch(); stopWatch.start(); + int totalRetryTime = 0; for (Map.Entry> entry : tableToParttions.entrySet()) { GetDeleteBitmapUpdateLockRequest.Builder builder = GetDeleteBitmapUpdateLockRequest.newBuilder(); builder.setTableId(entry.getKey()) @@ -790,10 +791,15 @@ private void getDeleteBitmapUpdateLock(Map> tableToParttions, lo cumulativePoints.put(tabletId, respCumulativePoints.get(i)); } } + totalRetryTime += retryTime; } stopWatch.stop(); - LOG.info("get delete bitmap lock successfully. txns: {}. time cost: {} ms.", - transactionId, stopWatch.getTime()); + if (totalRetryTime > 0 || stopWatch.getTime() > 20) { + LOG.info( + "get delete bitmap lock successfully. txns: {}. totalRetryTime: {}. " + + "partitionSize: {}. time cost: {} ms.", + transactionId, totalRetryTime, tableToParttions.size(), stopWatch.getTime()); + } } private void sendCalcDeleteBitmaptask(long dbId, long transactionId, From 4a72ddc1fe67e658597f9b8ab3d0e56767f7d08a Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 15 Jul 2024 19:43:26 +0800 Subject: [PATCH 4/7] [Fix](inverted index) fix fast execute for not_in expr (#37745) ## Proposed changes not_in predicate is not processed correctly in vin_predicate fast_execute --- be/src/vec/exprs/vexpr.cpp | 2 +- be/src/vec/functions/function.h | 2 +- .../inverted_index_p0/test_index_rqg_bug3.out | 43 ++++++++++ .../test_index_rqg_bug3.groovy | 81 +++++++++++++++++++ 4 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/inverted_index_p0/test_index_rqg_bug3.out create mode 100644 regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index d788c8dc518a79..64a4adfa6beeda 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -627,7 +627,7 @@ std::string VExpr::gen_predicate_result_sign(Block& block, const ColumnNumbers& std::string column_name = block.get_by_position(arguments[0]).name; pred_result_sign += BeConsts::BLOCK_TEMP_COLUMN_PREFIX + column_name + "_" + function_name + "_"; - if (function_name == "in") { + if (function_name == "in" || function_name == "not_in") { // Generating 'result_sign' from 'inlist' requires sorting the values. std::set values; for (size_t i = 1; i < arguments.size(); i++) { diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index f3af3870c2eb61..36558c11df0207 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -521,7 +521,7 @@ class DefaultFunction final : public IFunctionBase { auto function_name = function->get_name(); return function_name == "eq" || function_name == "ne" || function_name == "lt" || function_name == "gt" || function_name == "le" || function_name == "ge" || - function_name == "in"; + function_name == "in" || function_name == "not_in"; } Status eval_inverted_index(FunctionContext* context, diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug3.out b/regression-test/data/inverted_index_p0/test_index_rqg_bug3.out new file mode 100644 index 00000000000000..cd01bedc787d41 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_index_rqg_bug3.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_bug_1 -- +-10 2023-12-11 +-10 2023-12-12 +-10 2023-12-13 +-10 2023-12-15 +-10 2023-12-15 +-10 2023-12-19 +-10 2023-12-19 +-10 2024-01-17 +-10 2024-02-18 +-10 2024-02-18 +-10 2025-02-18 +-10 2026-01-18 +-10 2026-02-18 +-4 2023-12-10 +-4 2023-12-11 +-4 2023-12-16 +-4 2024-01-31 +0 2024-01-19 +1 2023-12-16 +1 2024-01-09 +2 2023-12-10 +2 2023-12-11 +2 2024-01-08 +2 2024-01-31 +3 2023-12-20 +3 2024-01-19 +3 2025-06-18 +3 2026-02-18 +3 2027-01-16 +4 2023-12-12 +4 2023-12-12 +4 2024-01-08 +5 2023-12-16 +6 2024-02-18 +7 2023-12-17 +7 2023-12-20 +7 2027-01-09 +8 2025-02-18 +9 2024-02-18 +9 2024-02-18 + diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy new file mode 100644 index 00000000000000..eb49100effd1ee --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_index_rqg_bug3", "test_index_rqg_bug3"){ + def table1 = "test_index_rqg_bug3" + + sql "drop table if exists ${table1}" + + sql """ + CREATE TABLE ${table1} ( + `col_int_undef_signed_not_null` INT NOT NULL, + `col_date_undef_signed_not_null` DATE NOT NULL, + `col_bigint_undef_signed_not_null_index_inverted` BIGINT NOT NULL, + `col_bigint_undef_signed_not_null` BIGINT NOT NULL, + `col_int_undef_signed` INT NULL, + `col_int_undef_signed_index_inverted` INT NULL, + `col_int_undef_signed_not_null_index_inverted` INT NOT NULL, + `col_bigint_undef_signed` BIGINT NULL, + `col_bigint_undef_signed_index_inverted` BIGINT NULL, + `col_date_undef_signed` DATE NULL, + `col_date_undef_signed_index_inverted` DATE NULL, + `col_date_undef_signed_not_null_index_inverted` DATE NOT NULL, + `col_varchar_10__undef_signed` VARCHAR(10) NULL, + `col_varchar_10__undef_signed_index_inverted` VARCHAR(10) NULL, + `col_varchar_10__undef_signed_not_null` VARCHAR(10) NOT NULL, + `col_varchar_10__undef_signed_not_null_index_inverted` VARCHAR(10) NOT NULL, + `col_varchar_1024__undef_signed` VARCHAR(1024) NULL, + `col_varchar_1024__undef_signed_index_inverted` VARCHAR(1024) NULL, + `col_varchar_1024__undef_signed_not_null` VARCHAR(1024) NOT NULL, + `col_varchar_1024__undef_signed_not_null_index_inverted` VARCHAR(1024) NOT NULL, + `pk` INT NULL, + INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, + INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_bigint_undef_signed_index_inverted_idx (`col_bigint_undef_signed_index_inverted`) USING INVERTED, + INDEX col_bigint_undef_signed_not_null_index_inverted_idx (`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_index_inverted_idx (`col_date_undef_signed_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_not_null_index_inverted_idx (`col_date_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_varchar_10__undef_signed_index_inverted_idx (`col_varchar_10__undef_signed_index_inverted`) USING INVERTED, + INDEX col_varchar_10__undef_signed_not_null_index_inverted_idx (`col_varchar_10__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_index_inverted_idx (`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx (`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED + ) ENGINE=OLAP + UNIQUE KEY(`col_int_undef_signed_not_null`, `col_date_undef_signed_not_null`, `col_bigint_undef_signed_not_null_index_inverted`, `col_bigint_undef_signed_not_null`) + DISTRIBUTED BY HASH(`col_bigint_undef_signed_not_null`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + insert into ${table1}(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_varchar_10__undef_signed,col_varchar_10__undef_signed_index_inverted,col_varchar_10__undef_signed_not_null,col_varchar_10__undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_index_inverted,col_varchar_1024__undef_signed_not_null,col_varchar_1024__undef_signed_not_null_index_inverted) values (0,null,null,6,-10,686522353588051391,null,-4348126053228614825,-6032791270525051561,'2025-06-18','2023-12-20','2024-02-18','2024-02-18',null,'o','o','x',null,null,'q','o'),(1,-4,-4,2,-4,-2326501514208488669,-8144130583099259882,6094773265621719520,-4389392644235126205,'2024-01-09','2023-12-19','2024-01-31','2024-02-18',null,null,'c','n',null,'k','p','y'),(2,-10,-4,-4,-10,-5761419801766661970,null,1575077084160592390,-6748295140918895470,'2023-12-09','2024-01-31','2023-12-11','2024-01-08','j',null,'m','k','p','s','s','u'),(3,2,null,3,6,7058901979703960583,-8791856880716210018,9136811854525385821,-584135196107353733,'2024-02-18','2023-12-14','2024-01-19','2024-02-18','e','h','i','b','x','a','h','d'),(4,6,-4,4,0,2965210993977402598,null,1623398766611349339,-719530465607559880,'2024-02-18','2024-01-31','2023-12-12','2027-01-16','c','p','h','o','t','l','t','k'),(5,-4,-10,7,-4,null,-4312428052005262901,1664741259066823538,-6520957605791451399,'2024-02-18','2024-01-19','2027-01-09','2024-01-31','f','d','q','k',null,'u','n','x'),(6,-10,0,4,-4,-4719935591389099252,-8933690584562378263,1354434296669241202,2684538782485650790,'2023-12-16','2026-01-18','2024-01-08','2025-06-18',null,null,'f','x','x','v','i','m'),(7,0,-10,-10,-10,-5480618991908294867,null,5334008298577672775,7173424974650158839,'2024-01-09',null,'2023-12-09','2024-02-18','y','k','x','u',null,null,'a','b'),(8,6,-4,5,8,-7188890154699493125,-1925845279956226794,-5657889190097714482,1640041513228273840,'2027-01-16',null,'2025-02-18','2023-12-20','y','x','p','w','d','j','k','d'),(9,null,-4,1,3,-2080159247468648985,-1306911382131817506,1219720796746600015,-978348523441835274,'2024-02-18','2025-06-18','2025-06-18','2025-02-18',null,'i','y','s','c',null,'m','o'),(10,2,1,-10,0,-7569151440408756108,null,1393963672572074196,-3822268135049166532,'2024-01-08','2027-01-16','2023-12-13','2024-02-18','s',null,'q','z',null,'u','b','v'),(11,9,8,-10,7,-4419333711868488787,5670069993430301249,-5101280938951897282,7291919080934920934,'2027-01-09','2025-02-18','2024-01-17','2026-01-18','u','c','d','r',null,'m','r','p'),(12,9,8,7,9,-697217673128595873,-2415793798160514654,-1909943068043737865,5844073448689265407,'2024-01-17','2025-02-17','2023-12-17','2023-12-14','a','z','v','o','q','u','z','h'),(13,-10,6,-10,1,null,null,-6933219731543784592,-4745778751501231119,'2023-12-20',null,'2026-01-18','2026-01-18',null,'r','w','c','k',null,'t','e'),(14,6,4,-10,-10,null,377907223978572611,-7447442174599113505,4949011330273779695,'2023-12-17','2024-02-18','2026-01-18','2024-01-08','g','w','u','k',null,'m','g','d'),(15,null,-4,-10,3,-5441857898369120483,-2001300041828347883,4385022502994073333,6762545521805735020,'2024-01-17','2023-12-11','2023-12-15','2023-12-10','p','u','p','i','i','c','j','v'),(16,8,1,-10,3,7499177078109776887,8002215544264694167,-4914597203639379766,7611185654075676834,null,null,'2024-01-08','2023-12-17','e','h','q','t',null,null,'p','d'),(17,-4,-10,3,6,null,null,4596273190394276006,-3248366019937329149,'2024-01-09','2023-12-19','2023-12-20','2023-12-11',null,null,'i','f',null,'t','q','a'),(18,0,-4,-10,1,null,-2000849949571150330,7208571222986345744,2598345170237057509,'2024-01-09','2023-12-09','2024-02-18','2024-01-19','a','f','w','o','b','y','q','f'),(19,7,7,-4,0,null,5717592572856392823,-8128856226419623044,-7534868883394863810,'2023-12-20','2023-12-09','2023-12-10','2023-12-12','r','t','y','d','l','a','y','v'),(20,2,null,0,2,null,-6905288165492491017,1934258578152616096,-1388806210542225140,null,null,'2024-01-19','2026-01-18','p',null,'p','g','u','b','i','c'),(21,null,null,-10,-4,1698759627767041241,null,-6613269394014189122,1915677852069340594,'2023-12-18','2024-01-31','2025-02-18','2024-01-08','e','g','l','h',null,null,'v','n'),(22,-10,-10,-4,5,-3720952595350369266,1539673860923570193,5089313038468606351,262016952853919148,'2023-12-15','2025-02-18','2024-01-19','2024-01-08','d','g','d','e','l',null,'m','g'),(23,-10,1,-10,7,null,-4884323809040291936,-4428424779275301738,-3325468851678420401,'2023-12-19','2027-01-16','2026-02-18','2024-01-09','z','v','v','v','r','d','j','y'),(24,4,8,3,-10,1026316126533561197,-8966784351064986909,496857885215447340,-6148636280121789215,'2024-02-18','2025-06-18','2026-02-18','2025-02-18','f',null,'j','k',null,'s','i','q'),(25,5,5,4,6,8574091287090543865,null,-773937635554104337,6026917236758217609,'2026-02-18','2027-01-09','2023-12-12','2024-01-19','s','p','t','d','t','l','u','m'),(26,0,-10,-10,9,-2429694321063869458,null,8908961259233183763,6894623222255264210,'2024-01-17','2023-12-14','2023-12-11','2023-12-09','w','o','l','g','m','r','h','i'),(27,5,0,2,-10,7748161344545453064,null,3244053576839674045,-7948008233666340932,'2024-02-18','2023-12-20','2024-01-08','2023-12-14','n','a','r','q','c','y','q','u'),(28,4,-4,5,4,null,2204997326584988589,7997961660331289189,8763906081360257030,'2025-06-18','2023-12-18','2023-12-16','2023-12-16','k','i','d','t','y','c','o','a'),(29,0,5,-4,4,null,null,7562098503580472041,929168144988769048,'2026-01-18','2023-12-11','2023-12-10','2024-01-31','p','d','j','j','j','h','f','p'),(30,5,-10,-4,4,3945007524910600918,null,-8466778503120120353,-9169615505661358433,'2023-12-13','2024-01-19','2023-12-16','2023-12-10',null,'p','g','d','e','e','r','u'),(31,5,4,8,6,-7544567449016691208,-7026985677386241453,-2698203866546802012,-8383194363413620107,'2024-01-09','2027-01-09','2025-02-18','2025-02-18','f','t','g','n','r','i','p','o'),(32,-4,null,7,-10,-5468978947503379654,-5676001133436456624,-5328902013300281884,2338117992866157501,'2023-12-20','2023-12-15','2023-12-20','2024-01-08','f','z','y','t','j','c','e','x'),(33,-10,6,-10,-10,6715916167220457165,-3864032264700366706,7115861918737510196,-937991761308321600,'2025-02-18','2024-02-18','2023-12-19','2024-02-18','x',null,'t','x','h','o','p','v'),(34,-4,8,9,-10,-4718823602932239136,-3633212199616285968,-5190227402771860745,5545611345261203982,'2024-01-08','2026-01-18','2024-02-18','2023-12-16','f','t','n','h',null,'y','e','u'),(35,-4,3,-10,8,null,7722389449895645140,-4965839022248115530,6494405496923526511,'2023-12-10','2024-02-18','2026-02-18','2024-01-09','u','t','a','t','w',null,'h','w'),(36,5,6,-10,0,null,null,84960662585385706,2611830596442233539,null,'2026-01-18','2023-12-15','2026-01-18','b','t','p','b','g','g','z','k'),(37,-10,-10,3,-10,null,-5462312886211329186,-2793882411087244594,7564921654378090880,'2025-06-18','2027-01-09','2027-01-16','2023-12-09','n','k','l','z','y','i','o','c'),(38,4,null,-10,3,null,2065313248357950073,2398686393322288278,-5793325226082177083,'2023-12-14','2024-01-17','2023-12-12','2024-01-31','m',null,'n','c','g','f','r','m'),(39,5,1,9,0,-2901110266746457515,-7419676417711330947,5568223068212783910,-8443206843568552423,'2023-12-20','2023-12-15','2024-02-18','2024-01-17','j',null,'x','m','c','u','j','a'),(40,-4,5,-4,-4,3686987810771014559,4528672918224579415,-531153650185309112,-4795413900154192584,'2023-12-12','2024-01-19','2024-01-31','2024-01-19','m','o','k','p','v','s','f','c'),(41,null,6,-10,-10,1371451390800312645,-945321182848207621,-8418988114521301883,-8987180461079691062,'2024-01-09','2023-12-10','2023-12-19','2023-12-12','s','i','x','u','h','e','q','y'),(42,-10,null,2,1,null,-2863490765313461654,3110048825870954129,-2547950560699735251,'2025-06-18','2024-01-08','2023-12-10','2023-12-10','d','y','d','h','t','o','t','w'),(43,3,0,4,1,-7282895869404970499,5532011705197458854,-4502369753730677912,-3032934141238479600,'2023-12-18','2024-02-18','2023-12-19','2026-02-18','w',null,'m','n','g',null,'j','q'),(44,null,null,2,9,5430716729257430348,null,-8208477558683957238,-7953995265596299120,'2023-12-18','2023-12-18','2023-12-11','2025-06-18','w','w','a','u','k','k','j','q'),(45,6,-4,-10,1,-8903356732633014005,null,2532821444113211740,-2346292639048659545,'2024-01-08','2023-12-12','2025-02-18','2023-12-19','v','b','k','e','i','q','h','l'),(46,7,-10,-10,-10,null,-6646527298990960109,-7898216427196445987,-1558528416630681469,null,'2027-01-09','2024-02-18','2025-02-17','k','d','o','n','h','g','x','p'),(47,3,6,3,7,-6864291355591117572,4024432796468900765,-6272917113481022986,-1984131539617763529,'2024-01-17','2024-01-17','2025-06-18','2025-06-18','c','p','t','y','i','c','y','i'),(48,9,null,1,-4,null,-7244007199905240117,8657019614874868097,-492287318340969091,'2024-02-18','2023-12-11','2024-01-09','2027-01-09',null,null,'m','i','t','k','r','x'),(49,1,null,1,5,8407263822602373073,-3275760834800206047,-2117832965174816037,5807219087033669504,'2023-12-13','2024-01-19','2023-12-16','2024-02-18','r','n','n','o','r','g','j','q'); + """ + + qt_select_bug_1 """ + SELECT col_int_undef_signed_not_null, col_date_undef_signed_not_null + FROM ${table1} AS table1 + WHERE ( + NOT ( + ( + table1.`col_int_undef_signed_not_null` is NULL + ) + OR table1.col_varchar_1024__undef_signed_index_inverted IN ('h', 'j') + ) + OR table1.`col_date_undef_signed_not_null` IN ('2024-02-18') + ) + ORDER BY col_int_undef_signed_not_null, col_date_undef_signed_not_null; """ + +} From 7919f0702e01cb42dabe7faf0ebfed086319fda1 Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Mon, 15 Jul 2024 19:43:51 +0800 Subject: [PATCH 5/7] [Bug](join) fix broadcast join running when hash table build not finished (#37792) in pr https://github.com/apache/doris/pull/37643/files wants fix the bug of join which not build hash table, but running early and not wait the finished signal. But this may be a normal phenomenon, as it should allows all sinks to run when the source operator have closed, so here return eof status directly when signal == false. --- be/src/pipeline/exec/analytic_source_operator.cpp | 5 ----- be/src/pipeline/exec/hashjoin_build_sink.cpp | 6 +++++- be/src/pipeline/exec/operator.cpp | 5 +++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/be/src/pipeline/exec/analytic_source_operator.cpp b/be/src/pipeline/exec/analytic_source_operator.cpp index 93e87cbce5d822..a036481d727789 100644 --- a/be/src/pipeline/exec/analytic_source_operator.cpp +++ b/be/src/pipeline/exec/analytic_source_operator.cpp @@ -559,11 +559,6 @@ Status AnalyticLocalState::close(RuntimeState* state) { std::vector tmp_result_window_columns; _result_window_columns.swap(tmp_result_window_columns); - // Some kinds of source operators has a 1-1 relationship with a sink operator (such as AnalyticOperator). - // We must ensure AnalyticSinkOperator will not be blocked if AnalyticSourceOperator already closed. - if (_shared_state && _shared_state->sink_deps.size() == 1) { - _shared_state->sink_deps.front()->set_always_ready(); - } return PipelineXLocalState::close(state); } diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp index 7887628b7fa476..3a55fdd9b8698e 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.cpp +++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp @@ -567,7 +567,11 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block* } else if (!local_state._should_build_hash_table) { DCHECK(_shared_hashtable_controller != nullptr); DCHECK(_shared_hash_table_context != nullptr); - CHECK(_shared_hash_table_context->signaled); + // the instance which is not build hash table, it's should wait the signal of hash table build finished. + // but if it's running and signaled == false, maybe the source operator have closed caused by some short circuit, + if (!_shared_hash_table_context->signaled) { + return Status::Error("source have closed"); + } if (!_shared_hash_table_context->status.ok()) { return _shared_hash_table_context->status; diff --git a/be/src/pipeline/exec/operator.cpp b/be/src/pipeline/exec/operator.cpp index eba380f4386aa3..ba3602a91cb1ef 100644 --- a/be/src/pipeline/exec/operator.cpp +++ b/be/src/pipeline/exec/operator.cpp @@ -511,6 +511,11 @@ Status PipelineXLocalState::close(RuntimeState* state) { _peak_memory_usage_counter->set(_mem_tracker->peak_consumption()); } _closed = true; + // Some kinds of source operators has a 1-1 relationship with a sink operator (such as AnalyticOperator). + // We must ensure AnalyticSinkOperator will not be blocked if AnalyticSourceOperator already closed. + if (_shared_state && _shared_state->sink_deps.size() == 1) { + _shared_state->sink_deps.front()->set_always_ready(); + } return Status::OK(); } From b8b36c6e7ccfaad96abe129ab138b3ab6915ea1f Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 15 Jul 2024 20:23:37 +0800 Subject: [PATCH 6/7] [test](fix) replace hardcode s3BucketName (#37739) ## Proposed changes Issue Number: close #xxx --------- Co-authored-by: stephen --- .../paimon/paimon_base_filesystem.groovy | 5 +- .../hive/test_hive_write_insert_s3.groovy | 5 +- .../suites/github_events_p2/load.groovy | 2 +- .../broker_load/test_compress_type.groovy | 41 ++-- ..._csv_with_enclose_and_escapeS3_load.groovy | 15 +- .../broker_load/test_etl_failed.groovy | 8 +- .../broker_load/test_multi_table_load.groovy | 15 +- .../load_p0/broker_load/test_s3_load.groovy | 44 ++-- .../load_p0/broker_load/test_seq_load.groovy | 7 +- .../broker_load/test_broker_load.groovy | 140 ++++++------- .../test_parquet_large_metadata_load.groovy | 15 +- .../test_s3_load_properties.groovy | 150 +++++++------- .../test_s3_load_with_load_parallelism.groovy | 7 +- .../tvf/test_tvf_based_broker_load.groovy | 74 +++---- .../suites/load_p2/tvf/test_s3_tvf.groovy | 190 +++++++++--------- .../stress_test_diff_date_list.groovy | 2 +- .../stress_test_same_date_range.groovy | 2 +- .../stress_test_two_stream_load.groovy | 2 +- .../query_profile/s3_load_profile_test.groovy | 8 +- ...pdate_rows_and_partition_first_load.groovy | 16 +- .../suites/tpcds_sf1000_p2/load.groovy | 5 +- 21 files changed, 389 insertions(+), 364 deletions(-) diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index 7be15f94243e7b..0e00cd8fb7a8bc 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -29,6 +29,7 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock String s3ak = getS3AK() String s3sk = getS3SK() + def s3Endpoint = getS3Endpoint() def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18""" def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a""" @@ -48,9 +49,9 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock create catalog if not exists ${catalog_oss} properties ( "type" = "paimon", "warehouse" = "oss://paimon-zd/paimonoss", - "oss.endpoint"="oss-cn-beijing.aliyuncs.com", "oss.access_key"="${ak}", - "oss.secret_key"="${sk}" + "oss.secret_key"="${sk}", + "oss.endpoint"="oss-cn-beijing.aliyuncs.com" ); """ logger.info("catalog " + catalog_cos + " created") diff --git a/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy b/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy index 87633ba1b09a5f..cf9fea67cbd685 100644 --- a/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy +++ b/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy @@ -17,6 +17,7 @@ suite("test_hive_write_insert_s3", "p2,external,hive,external_remote,external_remote_hive") { def format_compressions = ["parquet_snappy"] + def s3BucketName = getS3BucketName() def q01 = { String format_compression, String catalog_name -> logger.info("hive sql: " + """ truncate table all_types_${format_compression}_s3; """) @@ -76,8 +77,8 @@ suite("test_hive_write_insert_s3", "p2,external,hive,external_remote,external_re hive_remote """ DROP TABLE IF EXISTS all_types_par_${format_compression}_s3_${catalog_name}_q02; """ logger.info("hive sql: " + """ CREATE TABLE IF NOT EXISTS all_types_par_${format_compression}_s3_${catalog_name}_q02 like all_types_par_${format_compression}_s3; """) hive_remote """ CREATE TABLE IF NOT EXISTS all_types_par_${format_compression}_s3_${catalog_name}_q02 like all_types_par_${format_compression}_s3; """ - logger.info("hive sql: " + """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://doris-build-1308700295/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """) - hive_remote """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://doris-build-1308700295/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """ + logger.info("hive sql: " + """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://${s3BucketName}/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """) + hive_remote """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://${s3BucketName}/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """ sql """refresh catalog ${catalog_name};""" sql """ diff --git a/regression-test/suites/github_events_p2/load.groovy b/regression-test/suites/github_events_p2/load.groovy index dc2e0dbb97505c..92a588a2214b29 100644 --- a/regression-test/suites/github_events_p2/load.groovy +++ b/regression-test/suites/github_events_p2/load.groovy @@ -31,7 +31,7 @@ suite("load") { ak "${getS3AK()}" sk "${getS3SK()}" endpoint "http://${getS3Endpoint()}" - region "ap-beijing" + region "${getS3Region()}" repository "regression_test_github_events" snapshot "github_events" timestamp "2022-03-23-12-19-51" diff --git a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy index 693e533fa5086f..723a07d5296b37 100644 --- a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy +++ b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy @@ -17,6 +17,7 @@ suite("test_compress_type", "load_p0") { def tableName = "basic_data" + def s3BucketName = getS3BucketName() // GZ/LZO/BZ2/LZ4FRAME/DEFLATE/LZOP def compressTypes = [ @@ -62,24 +63,24 @@ suite("test_compress_type", "load_p0") { ] def paths = [ - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.gz", - "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.bz2", - "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.lz4", - "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.gz", - "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.bz2", - "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.lz4", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2", + "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4", + "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.gz", + "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.bz2", + "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.lz4", + "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.gz", + "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.bz2", + "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.lz4", ] def labels = [] @@ -137,8 +138,8 @@ suite("test_compress_type", "load_p0") { WITH S3 ( "AWS_ACCESS_KEY" = "$ak", "AWS_SECRET_KEY" = "$sk", - "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com", - "AWS_REGION" = "ap-beijing", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", "provider" = "${getS3Provider()}" ) """ diff --git a/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy b/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy index eea25fb453495f..291f623a512eac 100644 --- a/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy +++ b/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy @@ -19,6 +19,7 @@ suite("test_csv_with_enclose_and_escapeS3_load", "load_p0") { def tableName = "test_csv_with_enclose_and_escape" + def s3BucketName = getS3BucketName() sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ @@ -48,24 +49,24 @@ suite("test_csv_with_enclose_and_escapeS3_load", "load_p0") { ] for (i in 0.. 50", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv") .addProperty("format", "csv") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) @@ -720,14 +722,14 @@ suite("test_s3_tvf", "p2") { attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"WHERE c1 > 50", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv") .addProperty("format", "csv") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) for(String table : arrayTables) { attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "WHERE c1 > 50", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv") .addProperty("format", "csv") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) @@ -735,7 +737,7 @@ suite("test_s3_tvf", "p2") { for(String table : uniqTable) { attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "ORDER BY c1") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv") .addProperty("format", "csv") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) @@ -743,7 +745,7 @@ suite("test_s3_tvf", "p2") { for(String table : basicTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.parq") .addProperty("format", "parquet") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) @@ -751,14 +753,14 @@ suite("test_s3_tvf", "p2") { attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.parq") .addProperty("format", "parquet") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) for(String table : arrayTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.parq") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.parq") .addProperty("format", "parquet") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) @@ -766,7 +768,7 @@ suite("test_s3_tvf", "p2") { for(String table : basicTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.orc") .addProperty("format", "orc") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) @@ -774,14 +776,14 @@ suite("test_s3_tvf", "p2") { attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.orc") .addProperty("format", "orc") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) for(String table : arrayTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.orc") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.orc") .addProperty("format", "orc") .addProperty("column_separator", "|") .addProperty("force_parsing_by_standard_uri", "true")) @@ -789,7 +791,7 @@ suite("test_s3_tvf", "p2") { for(String table : basicTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json") .addProperty("format", "json") .addProperty("read_json_by_line", "false") .addProperty("strip_outer_array", "true") @@ -799,7 +801,7 @@ suite("test_s3_tvf", "p2") { attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json") .addProperty("format", "json") .addProperty("read_json_by_line", "false") .addProperty("strip_outer_array", "true") @@ -808,7 +810,7 @@ suite("test_s3_tvf", "p2") { for(String table : arrayTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.json") .addProperty("format", "json") .addProperty("read_json_by_line", "false") .addProperty("strip_outer_array", "true") @@ -818,7 +820,7 @@ suite("test_s3_tvf", "p2") { for(String table : basicTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json") .addProperty("format", "json") .addProperty("read_json_by_line", "true") .addProperty("strip_outer_array", "false") @@ -828,7 +830,7 @@ suite("test_s3_tvf", "p2") { attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json") .addProperty("format", "json") .addProperty("read_json_by_line", "true") .addProperty("strip_outer_array", "false") @@ -837,7 +839,7 @@ suite("test_s3_tvf", "p2") { for(String table : arrayTables) { attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_by_line.json") .addProperty("format", "json") .addProperty("read_json_by_line", "true") .addProperty("strip_outer_array", "false") @@ -850,7 +852,7 @@ suite("test_s3_tvf", "p2") { // line_delimiter: \t for(String table : basicTables) { attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line_delimiter.csv") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line_delimiter.csv") .addProperty("format", "csv") .addProperty("column_separator", "|") .addProperty("line_delimiter", "\t") @@ -859,7 +861,7 @@ suite("test_s3_tvf", "p2") { attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line_delimiter.csv") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line_delimiter.csv") .addProperty("format", "csv") .addProperty("column_separator", "|") .addProperty("line_delimiter", "\t") @@ -867,7 +869,7 @@ suite("test_s3_tvf", "p2") { for(String table : arrayTables) { attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "") - .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_tab_line_delimiter.csv") + .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_by_tab_line_delimiter.csv") .addProperty("format", "csv") .addProperty("column_separator", "|") .addProperty("line_delimiter", "\t") @@ -877,7 +879,7 @@ suite("test_s3_tvf", "p2") { // invalid line delimiter, this will case error // for(String table : basicTables) { // attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "") - // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv") + // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv") // .addProperty("format", "csv") // .addProperty("column_separator", "|") // .addProperty("line_delimiter", ",") @@ -886,7 +888,7 @@ suite("test_s3_tvf", "p2") { // attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21", // "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "") - // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv") + // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv") // .addProperty("format", "csv") // .addProperty("column_separator", "|") // .addProperty("line_delimiter", ",") @@ -894,7 +896,7 @@ suite("test_s3_tvf", "p2") { // for(String table : arrayTables) { // attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "") - // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv") + // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv") // .addProperty("format", "csv") // .addProperty("column_separator", "|") // .addProperty("line_delimiter", ",") @@ -923,7 +925,7 @@ suite("test_s3_tvf", "p2") { FROM S3 ( "s3.access_key" = "$ak", "s3.secret_key" = "$sk", - "s3.region" = "ap-beijing", + "s3.region" = "${s3Region}", ${prop} ) ${attribute.whereClause} ${attribute.orderByClause} diff --git a/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy b/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy index 2167c40f1d2a38..df7ce235e85afa 100644 --- a/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy +++ b/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy @@ -28,7 +28,7 @@ suite("stress_test_diff_date_list", "p2,nonConcurrent") { // get doris-db from s3 def dirPath = context.file.parent def fileName = "doris-dbgen" - def fileUrl = "http://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen" + def fileUrl = "http://${getS3BucketName()}.${getS3Endpoint()}/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen" def filePath = Paths.get(dirPath, fileName) if (!Files.exists(filePath)) { new URL(fileUrl).withInputStream { inputStream -> diff --git a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy index fb400105758601..c8bbdfbffc434f 100644 --- a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy +++ b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy @@ -28,7 +28,7 @@ suite("stress_test_same_date_range", "p2,nonConcurrent") { // get doris-db from s3 def dirPath = context.file.parent def fileName = "doris-dbgen" - def fileUrl = "http://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen" + def fileUrl = "http://${getS3BucketName()}.${getS3Endpoint()}/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen" def filePath = Paths.get(dirPath, fileName) if (!Files.exists(filePath)) { new URL(fileUrl).withInputStream { inputStream -> diff --git a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy index 478ee0d71240e5..ecbb277c5fd90b 100644 --- a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy +++ b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy @@ -26,7 +26,7 @@ suite("stress_test_two_stream_load", "p2,nonConcurrent") { // get doris-db from s3 def dirPath = context.file.parent def fileName = "doris-dbgen" - def fileUrl = "http://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen" + def fileUrl = "http://${getS3BucketName()}.${getS3Endpoint()}/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen" def filePath = Paths.get(dirPath, fileName) if (!Files.exists(filePath)) { new URL(fileUrl).withInputStream { inputStream -> diff --git a/regression-test/suites/query_profile/s3_load_profile_test.groovy b/regression-test/suites/query_profile/s3_load_profile_test.groovy index 33bdf48895a0d4..686e357d4e1395 100644 --- a/regression-test/suites/query_profile/s3_load_profile_test.groovy +++ b/regression-test/suites/query_profile/s3_load_profile_test.groovy @@ -29,6 +29,8 @@ def getProfile = { id -> // ref https://github.com/apache/doris/blob/3525a03815814f66ec78aa2ad6bbd9225b0e7a6b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy suite('s3_load_profile_test') { + def s3Endpoint = getS3Endpoint() + def s3Region = getS3Region() sql "drop table if exists dup_tbl_basic;" sql """ CREATE TABLE dup_tbl_basic @@ -97,7 +99,7 @@ PROPERTIES ( "replication_num" = "1" ); """ - def loadAttribute =new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", + def loadAttribute =new LoadAttributes("s3://${getS3BucketName()}/regression/load/data/basic_data.csv", "dup_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", "", "", "", "", "") @@ -128,8 +130,8 @@ PROPERTIES ( WITH S3 ( "AWS_ACCESS_KEY" = "$ak", "AWS_SECRET_KEY" = "$sk", - "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com", - "AWS_REGION" = "ap-beijing", + "AWS_ENDPOINT" = "${s3Endpoint}", + "AWS_REGION" = "${s3Region}", "use_path_style" = "$loadAttribute.usePathStyle", "provider" = "${getS3Provider()}" ) diff --git a/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy b/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy index a589a484d5ed15..ef927ddaf4329b 100644 --- a/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy +++ b/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy @@ -16,7 +16,9 @@ // under the License. suite("test_update_rows_and_partition_first_load", "p2") { - + def s3BucketName = getS3BucketName() + def s3Endpoint = getS3Endpoint() + def s3Region = getS3Region() String ak = getS3AK() String sk = getS3SK() String enabled = context.config.otherConfigs.get("enableBrokerLoad") @@ -88,24 +90,24 @@ suite("test_update_rows_and_partition_first_load", "p2") { def label = "part_" + UUID.randomUUID().toString().replace("-", "0") sql """ LOAD LABEL ${label} ( - DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_1.csv") + DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_1.csv") INTO TABLE update_rows_test1 COLUMNS TERMINATED BY ",", - DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_2.csv") + DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_2.csv") INTO TABLE update_rows_test2 COLUMNS TERMINATED BY ",", - DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_1.csv") + DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_1.csv") INTO TABLE partition_test1 COLUMNS TERMINATED BY ",", - DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_2.csv") + DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_2.csv") INTO TABLE partition_test2 COLUMNS TERMINATED BY "," ) WITH S3 ( "AWS_ACCESS_KEY" = "$ak", "AWS_SECRET_KEY" = "$sk", - "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com", - "AWS_REGION" = "ap-beijing", + "AWS_ENDPOINT" = "${s3Endpoint}", + "AWS_REGION" = "${s3Region}", "provider" = "${getS3Provider()}" ); """ diff --git a/regression-test/suites/tpcds_sf1000_p2/load.groovy b/regression-test/suites/tpcds_sf1000_p2/load.groovy index aaf4fd54d71466..9bf888e93b0d7b 100644 --- a/regression-test/suites/tpcds_sf1000_p2/load.groovy +++ b/regression-test/suites/tpcds_sf1000_p2/load.groovy @@ -21,12 +21,13 @@ * */ suite("load") { + def s3Region = getS3Region() restore { location "s3://${getS3BucketName()}/regression/tpcds/sf1000" ak "${getS3AK()}" sk "${getS3SK()}" endpoint "http://${getS3Endpoint()}" - region "ap-beijing" + region "${s3Region}" repository "tpcds_backup" snapshot "tpcds_customer" timestamp "2022-03-31-10-16-46" @@ -40,7 +41,7 @@ suite("load") { ak "${getS3AK()}" sk "${getS3SK()}" endpoint "http://${getS3Endpoint()}" - region "ap-beijing" + region "${s3Region}" repository "tpcds_backup" snapshot "tpcds" timestamp "2022-03-30-12-22-31" From 01f00aad6f1194d976d9a817a6a864ffbba28db4 Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 15 Jul 2024 21:20:33 +0800 Subject: [PATCH 7/7] [Bug](bitmap) clear set when bitmap fastunion (#37816) ## Proposed changes before:6 min 59.33 sec after :24.67 sec --- be/src/util/bitmap_value.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index ac5e826e1fee66..2d15ac99611274 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1252,8 +1252,7 @@ class BitmapValue { std::vector bitmaps; std::vector single_values; std::vector*> sets; - for (int i = 0; i < values.size(); ++i) { - auto* value = values[i]; + for (const auto* value : values) { switch (value->_type) { case EMPTY: break; @@ -1280,7 +1279,9 @@ class BitmapValue { _bitmap->add(_sv); break; case BITMAP: - *_bitmap |= detail::Roaring64Map::fastunion(bitmaps.size(), bitmaps.data()); + for (const auto* bitmap : bitmaps) { + *_bitmap |= *bitmap; + } break; case SET: { *_bitmap = detail::Roaring64Map::fastunion(bitmaps.size(), bitmaps.data()); @@ -1315,6 +1316,7 @@ class BitmapValue { _bitmap->add(v); } _type = BITMAP; + _set.clear(); break; case SET: { break;